{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.8.0\n",
      "1.14.0\n",
      "0.22.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/envs/py2env/lib/python2.7/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "# Import libraries and modules\n",
    "import sys\n",
    "import copy\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import shutil\n",
    "print tf.__version__\n",
    "print np.__version__\n",
    "print pd.__version__\n",
    "np.set_printoptions(threshold=np.inf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# change these to try this notebook out\n",
    "BUCKET = 'ryan-asl-energy-bucket'\n",
    "PROJECT = 'qwiklabs-gcp-8d3d0cd07cef9252'\n",
    "REGION = 'us-central1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Import os environment variables\n",
    "import os\n",
    "os.environ['BUCKET'] = BUCKET\n",
    "os.environ['PROJECT'] = PROJECT\n",
    "os.environ['REGION'] = REGION"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(filepath_or_buffer = \"../data/RealMarketPriceData.csv\", sep = \";\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>datetime</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01/01/2015 00:00</td>\n",
       "      <td>48.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01/01/2015 01:00</td>\n",
       "      <td>47.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01/01/2015 02:00</td>\n",
       "      <td>42.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01/01/2015 03:00</td>\n",
       "      <td>38.41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01/01/2015 04:00</td>\n",
       "      <td>35.72</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           datetime  price\n",
       "0  01/01/2015 00:00  48.10\n",
       "1  01/01/2015 01:00  47.33\n",
       "2  01/01/2015 02:00  42.27\n",
       "3  01/01/2015 03:00  38.41\n",
       "4  01/01/2015 04:00  35.72"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df['dayofweek'] = pd.DatetimeIndex(data = df['datetime']).dayofweek\n",
    "df['hourofday'] = pd.DatetimeIndex(data = df['datetime']).hour"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>datetime</th>\n",
       "      <th>price</th>\n",
       "      <th>dayofweek</th>\n",
       "      <th>hourofday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01/01/2015 00:00</td>\n",
       "      <td>48.10</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01/01/2015 01:00</td>\n",
       "      <td>47.33</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01/01/2015 02:00</td>\n",
       "      <td>42.27</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01/01/2015 03:00</td>\n",
       "      <td>38.41</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01/01/2015 04:00</td>\n",
       "      <td>35.72</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           datetime  price  dayofweek  hourofday\n",
       "0  01/01/2015 00:00  48.10          3          0\n",
       "1  01/01/2015 01:00  47.33          3          1\n",
       "2  01/01/2015 02:00  42.27          3          2\n",
       "3  01/01/2015 03:00  38.41          3          3\n",
       "4  01/01/2015 04:00  35.72          3          4"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df = df.drop(labels = ['datetime'], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "train_rows = int(len(df) * 0.8)\n",
    "traindf = df[:train_rows]\n",
    "evaldf = df[train_rows:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "          <script src=\"/static/components/requirejs/require.js\"></script>\n",
       "          <script>\n",
       "            requirejs.config({\n",
       "              paths: {\n",
       "                base: '/static/base',\n",
       "              },\n",
       "            });\n",
       "          </script>\n",
       "          "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "traindf.to_csv(path_or_buf = '../preproc/price_data_train.csv', header = False, index = False)\n",
    "evaldf.to_csv(path_or_buf = '../preproc/price_data_eval.csv', header = False, index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Local Development"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "arguments = {}\n",
    "arguments[\"train_file_pattern\"] = \"../preproc/price_data_train.csv\"\n",
    "arguments[\"eval_file_pattern\"] = \"../preproc/price_data_eval.csv\"\n",
    "arguments[\"output_dir\"] = \"trained_model\"\n",
    "arguments[\"batch_size\"] = 2\n",
    "arguments[\"input_sequence_length\"] = 5\n",
    "arguments[\"horizon\"] = 0\n",
    "arguments[\"output_sequence_length\"] = 4\n",
    "arguments[\"reverse_sequence\"] = False\n",
    "arguments[\"model_fn_name\"] = \"encoder_decoder_stacked_lstm_regression\"\n",
    "arguments[\"train_steps\"] = 1000\n",
    "arguments[\"lstm_hidden_units\"] = [64, 32, 16]\n",
    "arguments[\"lstm_dropout_output_keep_probs\"] = [0.95, 1.0, 1.0]\n",
    "arguments[\"dnn_hidden_units\"] = [1024, 256, 64]\n",
    "arguments[\"learning_rate\"] = 0.01\n",
    "arguments[\"start_delay_secs\"] = 60\n",
    "arguments[\"throttle_secs\"] = 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Set logging to be level of INFO\n",
    "tf.logging.set_verbosity(tf.logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Determine CSV and label columns\n",
    "CSV_COLUMNS = 'price,dayofweek,hourofday'.split(',')\n",
    "LABEL_COLUMN = 'price'\n",
    "\n",
    "# Set default values for each CSV column\n",
    "DEFAULTS = [[0.0], [0.0], [0.0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Create an input function reading a file using the Dataset API\n",
    "# Then provide the results to the Estimator API\n",
    "def read_dataset(filename, mode, batch_size, params):\n",
    "  def _input_fn():\n",
    "    print(\"\\nread_dataset: _input_fn: filename = {}\".format(filename))\n",
    "    print(\"read_dataset: _input_fn: mode = {}\".format(mode))\n",
    "    print(\"read_dataset: _input_fn: batch_size = {}\".format(batch_size))\n",
    "    print(\"read_dataset: _input_fn: params = {}\\n\".format(params))\n",
    "\n",
    "    def decode_csv(value_column):\n",
    "      print(\"\\nread_dataset: _input_fn: decode_csv: value_column = {}\".format(value_column))\n",
    "      columns = tf.decode_csv(records = value_column, record_defaults = DEFAULTS)\n",
    "      print(\"read_dataset: _input_fn: decode_csv: columns = {}\".format(columns))\n",
    "      features = dict(zip(CSV_COLUMNS, columns))\n",
    "      print(\"read_dataset: _input_fn: decode_csv: features = {}\".format(features))\n",
    "      return features\n",
    "    \n",
    "    def create_sequences(features):\n",
    "      # This function will create sequences out of the base features and labels by offsetting each sequence by one time step\n",
    "      print(\"\\nread_dataset: _input_fn: create_sequences: features = {}\".format(features))\n",
    "      \n",
    "      # These are our sequence lengths per batch, which is just our input_sequence_length tiled since all of our sequences are the same length\n",
    "      sequence_lengths = tf.tile(input = [params[\"input_sequence_length\"]], multiples = [params[\"batch_size\"]])\n",
    "      print(\"read_dataset: _input_fn: create_sequences: sequence_lengths = {}\".format(sequence_lengths))\n",
    "\n",
    "      # We will slice our label column since we will also be using the earlier values as a feature and the later values as the labels\n",
    "      label = features[LABEL_COLUMN][params['input_sequence_length'] + params['horizon']:] # shape = (batch_size,)\n",
    "      print(\"read_dataset: _input_fn: create_sequences: label = {}\".format(label))\n",
    "\n",
    "      # Now use tf.map_fn to break our labels and features into a params['batch_size'] number of sequences offset by one timestep\n",
    "      label = tf.map_fn(fn = lambda i: label[i:params['output_sequence_length'] + i], \n",
    "                        elems = tf.range(params['batch_size']), dtype=tf.float32) # shape = (params['batch_size'], params['input_sequence_length'])\n",
    "      print(\"read_dataset: _input_fn: create_sequences: label = {}\".format(label))\n",
    "      \n",
    "      for key,val in features.items():\n",
    "        features[key] = tf.map_fn(fn = lambda i: features[key][i:params['input_sequence_length'] + i], \n",
    "                                  elems = tf.range(params['batch_size']), dtype=tf.float32) # shape = (params['batch_size'], params['input_sequence_length'])\n",
    "        print(\"read_dataset: _input_fn: create_sequences: features[{}] = {}\".format(key, features[key]))\n",
    "        \n",
    "        # Reversing the input sequence can often improve performance because it shortens the path from the decoder to the relevant parts of the encoder\n",
    "        if params[\"reverse_sequence\"] == True:\n",
    "          features[key] = tf.reverse_sequence(input = features[key], seq_lengths = sequence_lengths, seq_axis = 1, batch_axis = 0)\n",
    "          print(\"read_dataset: _input_fn: create_sequences: reversed features[{}] = {}\".format(key, features[key]))\n",
    "\n",
    "      # Return our dictionary of feature tensors and our label tensor\n",
    "      return features, label\n",
    "\n",
    "    # Create list of files that match pattern\n",
    "    file_list = tf.gfile.Glob(filename = filename)\n",
    "    print(\"\\nread_dataset: _input_fn: file_list = {}\".format(file_list))\n",
    "\n",
    "    # Create dataset from file list\n",
    "    dataset = tf.data.TextLineDataset(filenames = file_list)  # Read text file\n",
    "    print(\"read_dataset: _input_fn: dataset.TextLineDataset(file_list) = {}\".format(dataset))\n",
    "\n",
    "    # Decode the CSV file into a features dictionary of tensors and a labels tensor\n",
    "    dataset = dataset.map(map_func = decode_csv)\n",
    "    print(\"read_dataset: _input_fn: dataset.map(decode_csv) = {}\".format(dataset))\n",
    "\n",
    "    # Determine amount of times to repeat file based on if we are training or evaluating\n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "      num_epochs = None # indefinitely\n",
    "    else:\n",
    "      num_epochs = 1 # end-of-input after this\n",
    "\n",
    "    # Repeat files num_epoch times\n",
    "    dataset = dataset.repeat(count = num_epochs)\n",
    "    print(\"read_dataset: _input_fn: dataset.repeat(num_epochs) = {}\".format(dataset))\n",
    "\n",
    "    # Group the data into batches\n",
    "    dataset = dataset.apply(transformation_func = tf.contrib.data.batch_and_drop_remainder(batch_size = batch_size))\n",
    "    print(\"read_dataset: _input_fn: dataset.batch(batch_size) = {}\".format(dataset))\n",
    "    \n",
    "    # Create sequences out of our batched data\n",
    "    dataset = dataset.map(map_func = create_sequences)\n",
    "    print(\"read_dataset: _input_fn: dataset.map(create_sequences) = {}\".format(dataset))\n",
    "\n",
    "    # Determine if we should shuffle based on if we are training or evaluating\n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "      dataset = dataset.shuffle(buffer_size = 10 * batch_size)\n",
    "      print(\"read_dataset: _input_fn: dataset.shuffle(buffer_size = 10 * batch_size) = {}\".format(dataset))\n",
    "\n",
    "    # Create a iterator and then pull the next batch of features and labels from the example queue\n",
    "    batch_features, batch_labels = dataset.make_one_shot_iterator().get_next()\n",
    "    print(\"read_dataset: _input_fn: batch_features = {}\".format(batch_features))\n",
    "    print(\"read_dataset: _input_fn: batch_labels = {}\\n\".format(batch_labels))\n",
    "\n",
    "    return batch_features, batch_labels\n",
    "  return _input_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def try_out_input_function(args):\n",
    "  with tf.Session() as sess:\n",
    "    fn = read_dataset(\n",
    "      filename = args['train_file_pattern'], \n",
    "      mode = tf.estimator.ModeKeys.EVAL, \n",
    "      batch_size = args['batch_size'] + args['input_sequence_length'] + args['horizon'] + args['output_sequence_length'] - 1,\n",
    "      params = args)\n",
    "    batch_features, batch_labels = fn()\n",
    "    features, labels = sess.run([batch_features, batch_labels])\n",
    "    print(\"try_out_input_function: features = {}\".format(features))\n",
    "    print(\"try_out_input_function: labels = {}\".format(labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "read_dataset: _input_fn: filename = ../preproc/price_data_train.csv\n",
      "read_dataset: _input_fn: mode = eval\n",
      "read_dataset: _input_fn: batch_size = 10\n",
      "read_dataset: _input_fn: params = {'throttle_secs': 30, 'hidden_units': [1024, 256, 64], 'eval_file_pattern': '../preproc/price_data_eval.csv', 'learning_rate': 0.01, 'start_delay_secs': 60, 'input_sequence_length': 5, 'train_steps': 1000, 'batch_size': 2, 'horizon': 0, 'train_file_pattern': '../preproc/price_data_train.csv', 'output_sequence_length': 4, 'output_dir': 'trained_model'}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = ['../preproc/price_data_train.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = <TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = Tensor(\"arg0:0\", shape=(), dtype=string)\n",
      "read_dataset: _input_fn: decode_csv: columns = [<tf.Tensor 'DecodeCSV:0' shape=() dtype=float32>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=float32>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=float32>]\n",
      "read_dataset: _input_fn: decode_csv: features = {'dayofweek': <tf.Tensor 'DecodeCSV:1' shape=() dtype=float32>, 'hourofday': <tf.Tensor 'DecodeCSV:2' shape=() dtype=float32>, 'price': <tf.Tensor 'DecodeCSV:0' shape=() dtype=float32>}\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = <MapDataset shapes: {dayofweek: (), hourofday: (), price: ()}, types: {dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = <RepeatDataset shapes: {dayofweek: (), hourofday: (), price: ()}, types: {dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = <_RestructuredDataset shapes: {dayofweek: (10,), hourofday: (10,), price: (10,)}, types: {dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}>\n",
      "\n",
      "read_dataset: _input_fn: create_sequences: features = {'dayofweek': <tf.Tensor 'arg0:0' shape=(10,) dtype=float32>, 'hourofday': <tf.Tensor 'arg1:0' shape=(10,) dtype=float32>, 'price': <tf.Tensor 'arg2:0' shape=(10,) dtype=float32>}\n",
      "read_dataset: _input_fn: dataset.map(create_sequences) = <MapDataset shapes: ({dayofweek: (2, ?), hourofday: (2, ?), price: (2, ?)}, (2, ?)), types: ({dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}, tf.float32)>\n",
      "read_dataset: _input_fn: batch_features = {'dayofweek': <tf.Tensor 'IteratorGetNext_2:0' shape=(2, ?) dtype=float32>, 'hourofday': <tf.Tensor 'IteratorGetNext_2:1' shape=(2, ?) dtype=float32>, 'price': <tf.Tensor 'IteratorGetNext_2:2' shape=(2, ?) dtype=float32>}\n",
      "read_dataset: _input_fn: batch_labels = Tensor(\"IteratorGetNext_2:3\", shape=(2, ?), dtype=float32)\n",
      "\n",
      "try_out_input_function: features = {'dayofweek': array([[3., 3., 3., 3., 3.],\n",
      "       [3., 3., 3., 3., 3.]], dtype=float32), 'hourofday': array([[0., 1., 2., 3., 4.],\n",
      "       [1., 2., 3., 4., 5.]], dtype=float32), 'price': array([[48.1 , 47.33, 42.27, 38.41, 35.72],\n",
      "       [47.33, 42.27, 38.41, 35.72, 35.13]], dtype=float32)}\n",
      "try_out_input_function: labels = [[35.13 36.22 32.4  36.6 ]\n",
      " [36.22 32.4  36.6  43.1 ]]\n"
     ]
    }
   ],
   "source": [
    "try_out_input_function(args = arguments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Create our model function to be used in our custom estimator\n",
    "def dnn_regression(features, labels, mode, params):\n",
    "  print(\"\\ndnn_regression: features = {}\".format(features))\n",
    "  print(\"dnn_regression: labels = {}\".format(labels))\n",
    "  print(\"dnn_regression: mode = {}\".format(mode))\n",
    "  print(\"dnn_regression: params = {}\".format(params))\n",
    "\n",
    "  # 0. Get tensor into correct shape\n",
    "  # Get dynamic batch size\n",
    "  current_batch_size = tf.shape(features[\"price\"])[0]\n",
    "  print(\"dnn_regression: current_batch_size = {}\".format(current_batch_size))\n",
    "  \n",
    "  # Get the number of features \n",
    "  number_of_features = len(features)\n",
    "  print(\"dnn_regression: number_of_features = {}\".format(number_of_features))\n",
    "  \n",
    "  # Stack all of the features into a 3-D tensor\n",
    "  X = tf.stack(features.values(), axis = 2) # shape = (batch_size, input_sequence_length, number_of_features)\n",
    "  print(\"dnn_regression: X = {}\".format(X))\n",
    "\n",
    "  # For DNN I want to flatten everything\n",
    "  X_flattened = tf.reshape(X, [current_batch_size, params['input_sequence_length'] * number_of_features]) # shape = (batch_size, input_sequence_length * number_of_features)\n",
    "  print(\"dnn_regression: X_flattened = {}\".format(X_flattened))\n",
    "\n",
    "  # 1. Create the DNN structure now\n",
    "  # Create the input layer to our frame DNN\n",
    "  network = X_flattened # shape = (batch_size, input_sequence_length * number_of_features)\n",
    "  print(\"dnn_regression: network = {}\".format(network))\n",
    "\n",
    "  # Add hidden layers with the given number of units/neurons per layer\n",
    "  for units in params['dnn_hidden_units']:\n",
    "    network = tf.layers.dense(inputs = network, units = units, activation = tf.nn.relu) # shape = (current_batch_size, dnn_hidden_units[i])\n",
    "    print(\"dnn_regression: network = {}, units = {}\".format(network, units))\n",
    "\n",
    "  # Connect the final hidden layer to a dense layer with no activation to get the predictions\n",
    "  predictions = tf.layers.dense(inputs = network, units = params['output_sequence_length'], activation = None) # shape = (current_batch_size, output_sequence_length)\n",
    "  print(\"dnn_regression: predictions = {}\\n\".format(predictions))\n",
    "\n",
    "  # 2. Loss function, training/eval ops\n",
    "  if mode == tf.estimator.ModeKeys.TRAIN or mode == tf.estimator.ModeKeys.EVAL:\n",
    "    loss = tf.losses.mean_squared_error(labels = labels, predictions = predictions)\n",
    "    train_op = tf.contrib.layers.optimize_loss(\n",
    "      loss = loss,\n",
    "      global_step = tf.train.get_global_step(),\n",
    "      learning_rate = params['learning_rate'],\n",
    "      optimizer = \"Adam\")\n",
    "    eval_metric_ops = {\n",
    "      \"rmse\": tf.metrics.root_mean_squared_error(labels = labels, predictions = predictions),\n",
    "      \"mae\": tf.metrics.mean_absolute_error(labels = labels, predictions = predictions)\n",
    "    }\n",
    "  else:\n",
    "    loss = None\n",
    "    train_op = None\n",
    "    eval_metric_ops = None\n",
    "\n",
    "  # 3. Create predictions\n",
    "  predictions_dict = {\"predicted\": predictions}\n",
    "\n",
    "  # 4. Create export outputs\n",
    "  export_outputs = {\"predict_export_outputs\": tf.estimator.export.PredictOutput(outputs = predictions)}\n",
    "\n",
    "  # 5. Return EstimatorSpec\n",
    "  return tf.estimator.EstimatorSpec(\n",
    "    mode = mode,\n",
    "    predictions = predictions_dict,\n",
    "    loss = loss,\n",
    "    train_op = train_op,\n",
    "    eval_metric_ops = eval_metric_ops,\n",
    "    export_outputs = export_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Create our model function to be used in our custom estimator\n",
    "def stacked_lstm_regression(features, labels, mode, params):\n",
    "  print(\"\\nstacked_lstm_regression: features = {}\".format(features))\n",
    "  print(\"stacked_lstm_regression: labels = {}\".format(labels))\n",
    "  print(\"stacked_lstm_regression: mode = {}\".format(mode))\n",
    "  print(\"stacked_lstm_regression: params = {}\".format(params))\n",
    "\n",
    "  # 0. Get tensor into correct shape\n",
    "  # Get dynamic batch size\n",
    "  current_batch_size = tf.shape(features[\"price\"])[0]\n",
    "  print(\"stacked_lstm_regression: current_batch_size = {}\".format(current_batch_size))\n",
    "  \n",
    "  # Get the number of features \n",
    "  number_of_features = len(features)\n",
    "  print(\"stacked_lstm_regression: number_of_features = {}\".format(number_of_features))\n",
    "  \n",
    "  # Stack all of the features into a 3-D tensor\n",
    "  X = tf.stack(values = features.values(), axis = 2) # shape = (current_batch_size, input_sequence_length, number_of_features)\n",
    "  print(\"stacked_lstm_regression: X = {}\".format(X))\n",
    "  \n",
    "  # Unstack all of 3-D features tensor into a sequence(list) of 2-D tensors of shape = (current_batch_size, number_of_features)\n",
    "  X_sequence = tf.unstack(value = X, num = params['input_sequence_length'], axis = 1)\n",
    "  print(\"stacked_lstm_regression: X_sequence = {}\".format(X_sequence))\n",
    "  \n",
    "  # 1. Configure the RNN\n",
    "  # First create a list of LSTM cells using our list of lstm hidden unit sizes\n",
    "  lstm_cells = [tf.contrib.rnn.BasicLSTMCell(num_units = units, forget_bias = 1.0, state_is_tuple = True) for units in params[\"lstm_hidden_units\"]] # list of LSTM cells\n",
    "  print(\"encoder_decoder_stacked_lstm_regression: lstm_cells = {}\".format(lstm_cells))\n",
    "  \n",
    "  # Next apply a dropout wrapper to our stack of LSTM cells, in this case just on the outputs\n",
    "  dropout_lstm_cells = [tf.nn.rnn_cell.DropoutWrapper(cell = lstm_cells[cell_index], \n",
    "                                                      input_keep_prob = 1.0, \n",
    "                                                      output_keep_prob = params[\"lstm_dropout_output_keep_probs\"][cell_index], \n",
    "                                                      state_keep_prob = 1.0) for cell_index in range(len(lstm_cells))]\n",
    "  print(\"encoder_decoder_stacked_lstm_regression: dropout_lstm_cells = {}\".format(dropout_lstm_cells))\n",
    "  \n",
    "  # Create a stack of layers of LSTM cells\n",
    "  stacked_lstm_cells = tf.contrib.rnn.MultiRNNCell(cells = dropout_lstm_cells, state_is_tuple = True) # combines list into MultiRNNCell object\n",
    "  print(\"encoder_decoder_stacked_lstm_regression: stacked_lstm_cells = {}\".format(stacked_lstm_cells))\n",
    "  \n",
    "  # Create a static RNN where we will keep a list of all of the intermediate hidden state outputs\n",
    "  outputs, _ = tf.nn.static_rnn(cell = stacked_lstm_cells, # creates list input_sequence_length elements long of 2-D tensors of shape = (current_batch_size, lstm_hidden_units[-1])\n",
    "                                inputs = X_sequence, \n",
    "                                initial_state = stacked_lstm_cells.zero_state(batch_size = current_batch_size, dtype = tf.float32), \n",
    "                                dtype = tf.float32)\n",
    "  print(\"stacked_lstm_regression: outputs = {}\".format(outputs))\n",
    "\n",
    "  # Slice outputs to keep only the last cell of the RNN\n",
    "  last_output = outputs[-1] # shape = (current_batch_size, lstm_hidden_units[-1])\n",
    "  print(\"stacked_lstm_regression: last_output = {}\".format(last_output))\n",
    "\n",
    "  # 2. Create the DNN structure now\n",
    "  # Create the input layer to our DNN\n",
    "  network = last_output # shape = (current_batch_size, lstm_hidden_units[-1])\n",
    "  print(\"stacked_lstm_regression: network = {}\".format(network))\n",
    "\n",
    "  # Add hidden layers with the given number of units/neurons per layer\n",
    "  for units in params['dnn_hidden_units']:\n",
    "    network = tf.layers.dense(inputs = network, units = units, activation = tf.nn.relu) # shape = (current_batch_size, dnn_hidden_units[i])\n",
    "    print(\"stacked_lstm_regression: network = {}, units = {}\".format(network, units))\n",
    "\n",
    "  # Connect the final hidden layer to a dense layer with no activation to get the predictions\n",
    "  predictions = tf.layers.dense(inputs = network, units = params['output_sequence_length'], activation = None) # shape = (current_batch_size, output_sequence_length)\n",
    "  print(\"stacked_lstm_regression: predictions = {}\\n\".format(predictions))\n",
    "\n",
    "  # 3. Loss function, training/eval ops\n",
    "  if mode == tf.estimator.ModeKeys.TRAIN or mode == tf.estimator.ModeKeys.EVAL:\n",
    "    loss = tf.losses.mean_squared_error(labels = labels, predictions = predictions)\n",
    "    train_op = tf.contrib.layers.optimize_loss(\n",
    "      loss = loss,\n",
    "      global_step = tf.train.get_global_step(),\n",
    "      learning_rate = params['learning_rate'],\n",
    "      optimizer = \"Adam\")\n",
    "    eval_metric_ops = {\n",
    "      \"rmse\": tf.metrics.root_mean_squared_error(labels = labels, predictions = predictions),\n",
    "      \"mae\": tf.metrics.mean_absolute_error(labels = labels, predictions = predictions)\n",
    "    }\n",
    "  else:\n",
    "    loss = None\n",
    "    train_op = None\n",
    "    eval_metric_ops = None\n",
    "\n",
    "  # 4. Create predictions\n",
    "  predictions_dict = {\"predicted\": predictions}\n",
    "\n",
    "  # 5. Create export outputs\n",
    "  export_outputs = {\"predict_export_outputs\": tf.estimator.export.PredictOutput(outputs = predictions)}\n",
    "\n",
    "  # 6. Return EstimatorSpec\n",
    "  return tf.estimator.EstimatorSpec(\n",
    "    mode = mode,\n",
    "    predictions = predictions_dict,\n",
    "    loss = loss,\n",
    "    train_op = train_op,\n",
    "    eval_metric_ops = eval_metric_ops,\n",
    "    export_outputs = export_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Create our model function to be used in our custom estimator\n",
    "def stacked_lstmN_regression(features, labels, mode, params):\n",
    "  print(\"\\nstacked_lstmN_regression: features = {}\".format(features))\n",
    "  print(\"stacked_lstmN_regression: labels = {}\".format(labels))\n",
    "  print(\"stacked_lstmN_regression: mode = {}\".format(mode))\n",
    "  print(\"stacked_lstmN_regression: params = {}\".format(params))\n",
    "\n",
    "  # 0. Get tensor into correct shape\n",
    "  # Get dynamic batch size\n",
    "  current_batch_size = tf.shape(features[\"price\"])[0]\n",
    "  print(\"stacked_lstmN_regression: current_batch_size = {}\".format(current_batch_size))\n",
    "  \n",
    "  # Get the number of features \n",
    "  number_of_features = len(features)\n",
    "  print(\"stacked_lstmN_regression: number_of_features = {}\".format(number_of_features))\n",
    "  \n",
    "  # Stack all of the features into a 3-D tensor\n",
    "  X = tf.stack(values = features.values(), axis = 2) # shape = (current_batch_size, input_sequence_length, number_of_features)\n",
    "  print(\"stacked_lstmN_regression: X = {}\".format(X))\n",
    "  \n",
    "  # Unstack all of 3-D features tensor into a sequence(list) of 2-D tensors of shape = (current_batch_size, number_of_features)\n",
    "  X_sequence = tf.unstack(value = X, num = params['input_sequence_length'], axis = 1)\n",
    "  print(\"stacked_lstmN_regression: X_sequence = {}\".format(X_sequence))\n",
    "  \n",
    "  # 1. Configure the RNN\n",
    "  # First create a list of LSTM cells using our list of lstm hidden unit sizes\n",
    "  lstm_cells = [tf.contrib.rnn.BasicLSTMCell(num_units = units, forget_bias = 1.0, state_is_tuple = True) for units in params[\"lstm_hidden_units\"]] # list of LSTM cells\n",
    "  print(\"stacked_lstm_regression: lstm_cells = {}\".format(lstm_cells))\n",
    "  \n",
    "  # Next apply a dropout wrapper to our stack of LSTM cells, in this case just on the outputs\n",
    "  dropout_lstm_cells = [tf.nn.rnn_cell.DropoutWrapper(cell = lstm_cells[cell_index], \n",
    "                                                      input_keep_prob = 1.0, \n",
    "                                                      output_keep_prob = params[\"lstm_dropout_output_keep_probs\"][cell_index], \n",
    "                                                      state_keep_prob = 1.0) for cell_index in range(len(lstm_cells))]\n",
    "  print(\"stacked_lstm_regression: dropout_lstm_cells = {}\".format(dropout_lstm_cells))\n",
    "  \n",
    "  # Create a stack of layers of LSTM cells\n",
    "  stacked_lstm_cells = tf.contrib.rnn.MultiRNNCell(cells = dropout_lstm_cells, state_is_tuple = True) # combines list into MultiRNNCell object\n",
    "  print(\"stacked_lstm_regression: stacked_lstm_cells = {}\".format(stacked_lstm_cells))\n",
    "  \n",
    "  # Create a static RNN where we will keep a list of all of the intermediate hidden state outputs\n",
    "  outputs, _ = tf.nn.static_rnn(cell = stacked_lstm_cells, # creates list input_sequence_length elements long of 2-D tensors of shape = (current_batch_size, lstm_hidden_units[-1])\n",
    "                                inputs = X_sequence, \n",
    "                                initial_state = stacked_lstm_cells.zero_state(batch_size = current_batch_size, dtype = tf.float32), \n",
    "                                dtype = tf.float32)\n",
    "  print(\"stacked_lstmN_regression: outputs = {}\".format(outputs))\n",
    "\n",
    "  # Stack all of outputs into a 3-D tensor\n",
    "  stacked_outputs = tf.stack(values = outputs, axis = 1) # shape = (current_batch_size, input_sequence_length, lstm_hidden_units[-1])\n",
    "  print(\"stacked_lstmN_regression: stacked_outputs = {}\".format(stacked_outputs))\n",
    "  \n",
    "  # Flatten stacked outputs into a 2-D tensor\n",
    "  flattened_stacked_outputs = tf.reshape(tensor = stacked_outputs, shape = [current_batch_size, params['input_sequence_length'] * params['lstm_hidden_units'][-1]]) # shape = (current_batch_size, input_sequence_length * lstm_hidden_units[-1])\n",
    "  print(\"stacked_lstmN_regression: flattened_stacked_outputs = {}\".format(flattened_stacked_outputs))\n",
    "\n",
    "  # 2. Create the DNN structure now\n",
    "  # Create the input layer to our DNN\n",
    "  network = flattened_stacked_outputs # shape = (current_batch_size, input_sequence_length * lstm_hidden_units[-1])\n",
    "  print(\"stacked_lstmN_regression: network = {}\".format(network))\n",
    "\n",
    "  # Add hidden layers with the given number of units/neurons per layer\n",
    "  for units in params['dnn_hidden_units']:\n",
    "    network = tf.layers.dense(inputs = network, units = units, activation = tf.nn.relu) # shape = (current_batch_size, dnn_hidden_units[i])\n",
    "    print(\"stacked_lstmN_regression: network = {}, units = {}\".format(network, units))\n",
    "\n",
    "  # Connect the final hidden layer to a dense layer with no activation to get the predictions\n",
    "  predictions = tf.layers.dense(inputs = network, units = params['output_sequence_length'], activation = None) # shape = (current_batch_size, output_sequence_length)\n",
    "  print(\"stacked_lstmN_regression: predictions = {}\\n\".format(predictions))\n",
    "\n",
    "  # 3. Loss function, training/eval ops\n",
    "  if mode == tf.estimator.ModeKeys.TRAIN or mode == tf.estimator.ModeKeys.EVAL:\n",
    "    loss = tf.losses.mean_squared_error(labels = labels, predictions = predictions)\n",
    "    train_op = tf.contrib.layers.optimize_loss(\n",
    "      loss = loss,\n",
    "      global_step = tf.train.get_global_step(),\n",
    "      learning_rate = params['learning_rate'],\n",
    "      optimizer = \"Adam\")\n",
    "    eval_metric_ops = {\n",
    "      \"rmse\": tf.metrics.root_mean_squared_error(labels = labels, predictions = predictions),\n",
    "      \"mae\": tf.metrics.mean_absolute_error(labels = labels, predictions = predictions)\n",
    "    }\n",
    "  else:\n",
    "    loss = None\n",
    "    train_op = None\n",
    "    eval_metric_ops = None\n",
    "\n",
    "  # 4. Create predictions\n",
    "  predictions_dict = {\"predicted\": predictions}\n",
    "\n",
    "  # 5. Create export outputs\n",
    "  export_outputs = {\"predict_export_outputs\": tf.estimator.export.PredictOutput(outputs = predictions)}\n",
    "\n",
    "  # 6. Return EstimatorSpec\n",
    "  return tf.estimator.EstimatorSpec(\n",
    "    mode = mode,\n",
    "    predictions = predictions_dict,\n",
    "    loss = loss,\n",
    "    train_op = train_op,\n",
    "    eval_metric_ops = eval_metric_ops,\n",
    "    export_outputs = export_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Create our model function to be used in our custom estimator\n",
    "def encoder_decoder_stacked_lstm_regression(features, labels, mode, params):\n",
    "  print(\"\\nencoder_decoder_stacked_lstm_regression: features = {}\".format(features))\n",
    "  print(\"encoder_decoder_stacked_lstm_regression: labels = {}\".format(labels)) # shape = (current_batch_size, output_sequence_length)\n",
    "  print(\"encoder_decoder_stacked_lstm_regression: mode = {}\".format(mode))\n",
    "  print(\"encoder_decoder_stacked_lstm_regression: params = {}\".format(params))\n",
    "\n",
    "  # 0. Get input sequence tensor into correct shape\n",
    "  # Get dynamic batch size in case there was a partially filled batch\n",
    "  current_batch_size = tf.shape(features[\"price\"])[0]\n",
    "  print(\"encoder_decoder_stacked_lstm_regression: current_batch_size = {}\".format(current_batch_size))\n",
    "  \n",
    "  # Get the number of features \n",
    "  number_of_features = len(features)\n",
    "  print(\"encoder_decoder_stacked_lstm_regression: number_of_features = {}\".format(number_of_features))\n",
    "  \n",
    "  # Stack all of the features into a 3-D tensor\n",
    "  X = tf.stack(values = features.values(), axis = 2) # shape = (current_batch_size, input_sequence_length, number_of_features)\n",
    "  print(\"encoder_decoder_stacked_lstm_regression: X = {}\".format(X))\n",
    "  \n",
    "  # Unstack all of 3-D features tensor into a sequence(list) of 2-D tensors of shape = (current_batch_size, number_of_features)\n",
    "  X_sequence = tf.unstack(value = X, num = params['input_sequence_length'], axis = 1)\n",
    "  print(\"encoder_decoder_stacked_lstm_regression: X_sequence = {}\".format(X_sequence))\n",
    "\n",
    "  ################################################################################\n",
    "  \n",
    "  # 1. Create encoder of encoder-decoder LSTM stacks\n",
    "  # First create a list of LSTM cells using our list of lstm hidden unit sizes\n",
    "  lstm_cells = [tf.contrib.rnn.BasicLSTMCell(num_units = units, forget_bias = 1.0, state_is_tuple = True) for units in params[\"lstm_hidden_units\"]] # list of LSTM cells\n",
    "  print(\"encoder_decoder_stacked_lstm_regression: lstm_cells = {}\".format(lstm_cells))\n",
    "  \n",
    "  # Next apply a dropout wrapper to our stack of LSTM cells, in this case just on the outputs\n",
    "  dropout_lstm_cells = [tf.nn.rnn_cell.DropoutWrapper(cell = lstm_cells[cell_index], \n",
    "                                                      input_keep_prob = 1.0, \n",
    "                                                      output_keep_prob = params[\"lstm_dropout_output_keep_probs\"][cell_index], \n",
    "                                                      state_keep_prob = 1.0) for cell_index in range(len(lstm_cells))]\n",
    "  print(\"encoder_decoder_stacked_lstm_regression: dropout_lstm_cells = {}\".format(dropout_lstm_cells))\n",
    "  \n",
    "  # Create a stack of layers of LSTM cells\n",
    "  stacked_lstm_cells = tf.contrib.rnn.MultiRNNCell(cells = dropout_lstm_cells, state_is_tuple = True) # combines list into MultiRNNCell object\n",
    "  print(\"encoder_decoder_stacked_lstm_regression: stacked_lstm_cells = {}\".format(stacked_lstm_cells))\n",
    "  \n",
    "  # Create the encoder variable scope\n",
    "  with tf.variable_scope(\"encoder\"):\n",
    "    # Clone the stacked_lstm_cells subgraph since we will be using a copy for the encoder side and the decoder side\n",
    "    encoder_cells = copy.deepcopy(stacked_lstm_cells)\n",
    "    print(\"encoder_decoder_stacked_lstm_regression: encoder_cells = {}\".format(encoder_cells))\n",
    "    \n",
    "    # Encode the input sequence using our encoder stack of LSTMs\n",
    "    encoder_outputs, encoder_final_state = tf.nn.static_rnn(cell = encoder_cells, inputs = X_sequence, dtype = tf.float32)\n",
    "    print(\"encoder_decoder_stacked_lstm_regression: encoder_outputs = {}\".format(encoder_outputs)) # list input_sequence_length long of shape = (current_batch_size, lstm_hidden_units[-1])\n",
    "    print(\"encoder_decoder_stacked_lstm_regression: encoder_final_state = {}\".format(encoder_final_state)) # tuple of final encoder c_state and h_state\n",
    "\n",
    "  ################################################################################\n",
    "\n",
    "  # 2. Create decoder of encoder-decoder LSTM stacks\n",
    "  # The rnn_decoder function takes labels during TRAIN/EVAL and a start token followed by its previous predictions during PREDICT\n",
    "  # Starts with an intial state of the final encoder states\n",
    "  def rnn_decoder(decoder_inputs, initial_state, cell, inference):\n",
    "    # Create the decoder variable scope\n",
    "    with tf.variable_scope(\"decoder\"):\n",
    "      # Load in our initial state from our encoder\n",
    "      state = initial_state # tuple of final encoder c_state and h_state\n",
    "      print(\"encoder_decoder_stacked_lstm_regression: rnn_decoder: state = {}\".format(state))\n",
    "      \n",
    "      # Create an empty list to store our hidden state output for every timestep\n",
    "      outputs = []\n",
    "      \n",
    "      # Begin with no previous output\n",
    "      previous_output = None\n",
    "      \n",
    "      # Loop over all of our decoder_inputs which will be output_sequence_length long\n",
    "      for index, decoder_input in enumerate(decoder_inputs):\n",
    "        # If there has been a previous output then we will determine the next input\n",
    "        if previous_output is not None:\n",
    "          # Create the input layer to our DNN\n",
    "          network = previous_output # shape = (current_batch_size, lstm_hidden_units[-1])\n",
    "          print(\"encoder_decoder_stacked_lstm_regression: rnn_decoder: network = {}\".format(network))\n",
    "          \n",
    "          # Create our dnn variable scope\n",
    "          with tf.variable_scope(name_or_scope = \"dnn\", reuse = tf.AUTO_REUSE):\n",
    "            # Add hidden layers with the given number of units/neurons per layer\n",
    "            for units in params['dnn_hidden_units']:\n",
    "              network = tf.layers.dense(inputs = network, units = units, activation = tf.nn.relu) # shape = (current_batch_size, dnn_hidden_units[i])\n",
    "              print(\"encoder_decoder_stacked_lstm_regression: rnn_decoder: network = {}, units = {}\".format(network, units))\n",
    "              \n",
    "            # Connect the final hidden layer to a dense layer with no activation to get the logits\n",
    "            logits = tf.layers.dense(inputs = network, units = 1, activation = None) # shape = (current_batch_size, 1)\n",
    "            print(\"encoder_decoder_stacked_lstm_regression: rnn_decoder: logits = {}\\n\".format(logits))\n",
    "          \n",
    "          # If we are in inference then we will overwrite our next decoder_input with the logits we just calculated.\n",
    "          # Otherwise, we leave the decoder_input input as it was from the enumerated list\n",
    "          # We have to calculate the logits even when not using them so that the correct dnn subgraph will be generated here and after the encoder-decoder for both training and inference\n",
    "          if inference == True:\n",
    "            decoder_input = logits # shape = (current_batch_size, 1)\n",
    "\n",
    "          print(\"encoder_decoder_stacked_lstm_regression: rnn_decoder: decoder_input = {}\\n\".format(decoder_input))\n",
    "        \n",
    "        # If this isn't our first time through the loop, just reuse(share) the same variables for each iteration within the current variable scope\n",
    "        if index > 0:\n",
    "          tf.get_variable_scope().reuse_variables()\n",
    "        \n",
    "        # Run the decoder input through the decoder stack picking up from the previous state\n",
    "        output, state = cell(decoder_input, state)\n",
    "        print(\"encoder_decoder_stacked_lstm_regression: rnn_decoder: output = {}\".format(output)) # shape = (current_batch_size, lstm_hidden_units[-1])\n",
    "        print(\"encoder_decoder_stacked_lstm_regression: rnn_decoder: state = {}\".format(state)) # tuple of final decoder c_state and h_state\n",
    "        \n",
    "        # Append the current decoder hidden state output to the outputs list\n",
    "        outputs.append(output) # growing list eventually output_sequence_length long of shape = (current_batch_size, lstm_hidden_units[-1])\n",
    "        \n",
    "        # Set the previous output to the output just calculated\n",
    "        previous_output = output # shape = (current_batch_size, lstm_hidden_units[-1])\n",
    "    return outputs, state\n",
    "  \n",
    "  # Encoder-decoders work differently during training/evaluation and inference so we will have two separate subgraphs for each\n",
    "  if mode == tf.estimator.ModeKeys.TRAIN  or mode == tf.estimator.ModeKeys.EVAL:\n",
    "    # Break 2-D labels tensor into a list of 1-D tensors\n",
    "    unstacked_labels = tf.unstack(value = labels, num = params['output_sequence_length'], axis = 1) # list of output_sequence_length long of shape = (current_batch_size,)\n",
    "    print(\"encoder_decoder_stacked_lstm_regression: unstacked_labels = {}\".format(unstacked_labels))\n",
    "\n",
    "    # Expand each 1-D label tensor back into a 2-D tensor\n",
    "    expanded_unstacked_labels = [tf.expand_dims(input = tensor, axis = -1) for tensor in unstacked_labels] # list of output_sequence_length long of shape = (current_batch_size, 1)\n",
    "    print(\"encoder_decoder_stacked_lstm_regression: expanded_unstacked_labels = {}\".format(expanded_unstacked_labels))\n",
    "    \n",
    "    # Call our decoder using the labels as our inputs, the encoder final state as our initial state, our other LSTM stack as our cells, and inference set to false\n",
    "    decoder_outputs, decoder_states = rnn_decoder(decoder_inputs = expanded_unstacked_labels, initial_state = encoder_final_state, cell = stacked_lstm_cells, inference = False)\n",
    "  else:\n",
    "    # Since this is inference create fake labels. The list length needs to be the output sequence length even though only the first element is actually used (as our go signal)\n",
    "    fake_labels = [tf.zeros(shape = [current_batch_size, 1]) for _ in range(params['output_sequence_length'])]\n",
    "    print(\"encoder_decoder_stacked_lstm_regression: fake_labels = {}\".format(fake_labels))\n",
    "    \n",
    "    # Call our decoder using fake labels as our inputs, the encoder final state as our initial state, our other LSTM stack as our cells, and inference set to true\n",
    "    decoder_outputs, decoder_states = rnn_decoder(decoder_inputs = fake_labels, initial_state = encoder_final_state, cell = stacked_lstm_cells, inference = True)\n",
    "  print(\"encoder_decoder_stacked_lstm_regression: decoder_outputs = {}\".format(decoder_outputs)) # list output_sequence_length long of shape = (current_batch_size, lstm_hidden_units[-1])\n",
    "  print(\"encoder_decoder_stacked_lstm_regression: decoder_states = {}\".format(decoder_states)) # tuple of final decoder c_state and h_state\n",
    "  \n",
    "  # Stack together the list of decoder output tensors into one \n",
    "  stacked_decoder_outputs = tf.stack(values = decoder_outputs, axis = 0) # shape = (current_batch_size * output_sequence_length, lstm_hidden_units[-1])\n",
    "  print(\"encoder_decoder_stacked_lstm_regression: stacked_decoder_outputs = {}\".format(stacked_decoder_outputs))\n",
    "  \n",
    "  ################################################################################\n",
    "  \n",
    "  # 3. Create the DNN structure now after the encoder-decoder LSTM stack\n",
    "  # Create the input layer to our DNN\n",
    "  network = stacked_decoder_outputs # shape = (current_batch_size * output_sequence_length, lstm_hidden_units[-1])\n",
    "  print(\"encoder_decoder_stacked_lstm_regression: network = {}\".format(network))\n",
    "  \n",
    "  # Reuse the same variable scope as we used within our decoder (for inference)\n",
    "  with tf.variable_scope(name_or_scope = \"dnn\", reuse = tf.AUTO_REUSE):\n",
    "    # Add hidden layers with the given number of units/neurons per layer\n",
    "    for units in params['dnn_hidden_units']:\n",
    "      network = tf.layers.dense(inputs = network, units = units, activation = tf.nn.relu) # shape = (current_batch_size * output_sequence_length, dnn_hidden_units[i])\n",
    "      print(\"encoder_decoder_stacked_lstm_regression: network = {}, units = {}\".format(network, units))\n",
    "\n",
    "    # Connect the final hidden layer to a dense layer with no activation to get the logits\n",
    "    logits = tf.layers.dense(inputs = network, units = 1, activation = None) # shape = (current_batch_size * output_sequence_length, 1)\n",
    "    print(\"encoder_decoder_stacked_lstm_regression: logits = {}\\n\".format(logits))\n",
    "  \n",
    "  # Now that we are through the final DNN for each sequence element for each example in the batch, reshape the predictions to match our labels\n",
    "  predictions = tf.reshape(tensor = logits, shape = [current_batch_size, params['output_sequence_length']]) # shape = (current_batch_size, output_sequence_length)\n",
    "  print(\"encoder_decoder_stacked_lstm_regression: predictions = {}\\n\".format(predictions))\n",
    "\n",
    "  # 3. Loss function, training/eval ops\n",
    "  if mode == tf.estimator.ModeKeys.TRAIN or mode == tf.estimator.ModeKeys.EVAL:\n",
    "    loss = tf.losses.mean_squared_error(labels = labels, predictions = predictions)\n",
    "    train_op = tf.contrib.layers.optimize_loss(\n",
    "      loss = loss,\n",
    "      global_step = tf.train.get_global_step(),\n",
    "      learning_rate = params['learning_rate'],\n",
    "      optimizer = \"Adam\")\n",
    "    eval_metric_ops = {\n",
    "      \"rmse\": tf.metrics.root_mean_squared_error(labels = labels, predictions = predictions),\n",
    "      \"mae\": tf.metrics.mean_absolute_error(labels = labels, predictions = predictions)\n",
    "    }\n",
    "  else:\n",
    "    loss = None\n",
    "    train_op = None\n",
    "    eval_metric_ops = None\n",
    "\n",
    "  # 4. Create predictions\n",
    "  predictions_dict = {\"predicted\": predictions}\n",
    "\n",
    "  # 5. Create export outputs\n",
    "  export_outputs = {\"predict_export_outputs\": tf.estimator.export.PredictOutput(outputs = predictions)}\n",
    "\n",
    "  # 6. Return EstimatorSpec\n",
    "  return tf.estimator.EstimatorSpec(\n",
    "    mode = mode,\n",
    "    predictions = predictions_dict,\n",
    "    loss = loss,\n",
    "    train_op = train_op,\n",
    "    eval_metric_ops = eval_metric_ops,\n",
    "    export_outputs = export_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Create our serving input function to accept the data at serving and send it in the right format to our custom estimator\n",
    "def serving_input_fn(input_sequence_length, reverse_sequence):\n",
    "  # This function fixes the shape and type of our input strings\n",
    "  def fix_shape_and_type_for_serving(placeholder):\n",
    "    # String split each string in the batch and output the values from the resulting SparseTensors\n",
    "    split_string = tf.map_fn(\n",
    "      fn = lambda x: tf.string_split(source = [placeholder[x]], delimiter=',').values, \n",
    "      elems = tf.range(start = 0, limit = tf.shape(input = placeholder)[0]), \n",
    "      dtype = tf.string) # shape = (batch_size, input_sequence_length)\n",
    "    print(\"serving_input_fn: fix_shape_and_type_for_serving: split_string = {}\".format(split_string))\n",
    "\n",
    "    # Convert each string in the split tensor to float\n",
    "    feature_tensor = tf.string_to_number(string_tensor = split_string, out_type = tf.float32) # shape = (batch_size, input_sequence_length)\n",
    "    print(\"serving_input_fn: fix_shape_and_type_for_serving: feature_tensor = {}\".format(feature_tensor))\n",
    "    return feature_tensor\n",
    "  \n",
    "  # This function fixes dynamic shape ambiguity of last dimension so that we will be able to use it in our DNN (since tf.layers.dense require the last dimension to be known)\n",
    "  def get_shape_and_set_modified_shape_2D(tensor, additional_dimension_sizes):\n",
    "    # Get static shape for tensor and convert it to list\n",
    "    shape = tensor.get_shape().as_list()\n",
    "    # Set outer shape to additional_dimension_sizes[0] since we know that this is the correct size\n",
    "    shape[1] = additional_dimension_sizes[0]\n",
    "    # Set the shape of tensor to our modified shape\n",
    "    tensor.set_shape(shape = shape) # shape = (batch_size, additional_dimension_sizes[0])\n",
    "    print(\"serving_input_fn: get_shape_and_set_modified_shape_2D: tensor = {}, additional_dimension_sizes = {}\".format(tensor, additional_dimension_sizes))\n",
    "    return tensor\n",
    "      \n",
    "  # Create placeholders to accept the data sent to the model at serving time\n",
    "  feature_placeholders = { # all features come in as a batch of strings, shape = (batch_size,), this was so because of passing the arrays to online ml-engine prediction\n",
    "    'price': tf.placeholder(dtype = tf.string, shape = [None]),\n",
    "    'dayofweek': tf.placeholder(dtype = tf.string, shape = [None]),\n",
    "    'hourofday': tf.placeholder(dtype = tf.string, shape = [None])\n",
    "  }\n",
    "  print(\"\\nserving_input_fn: feature_placeholders = {}\".format(feature_placeholders))\n",
    "  \n",
    "  # Create feature tensors\n",
    "  features = {key: fix_shape_and_type_for_serving(placeholder = tensor) for key, tensor in feature_placeholders.items()}\n",
    "  print(\"serving_input_fn: features = {}\".format(features))\n",
    "  \n",
    "  # Fix dynamic shape ambiguity of feature tensors for our DNN\n",
    "  features = {key: get_shape_and_set_modified_shape_2D(tensor = tensor, additional_dimension_sizes = [input_sequence_length]) for key, tensor in features.items()}\n",
    "  print(\"serving_input_fn: features = {}\".format(features))\n",
    "\n",
    "  # These are our sequence lengths per batch, which is just our input_sequence_length tiled since all of our sequences are the same length\n",
    "  sequence_lengths = tf.tile(input = [input_sequence_length], multiples = [tf.shape(feature_placeholders[\"price\"])[0]])\n",
    "  print(\"serving_input_fn: sequence_lengths = {}\".format(sequence_lengths))\n",
    "\n",
    "  # Reversing the input sequence can often improve performance because it shortens the path from the decoder to the relevant parts of the encoder\n",
    "  if reverse_sequence == True:\n",
    "    features = {key: tf.reverse_sequence(input = tensor, seq_lengths = sequence_lengths, seq_axis = 1, batch_axis = 0) for key, tensor in features.items()}\n",
    "  else:\n",
    "    features = {key: tensor for key, tensor in features.items()}\n",
    "  print(\"serving_input_fn: features = {}\\n\".format(features))\n",
    "        \n",
    "  return tf.estimator.export.ServingInputReceiver(features = features, receiver_tensors = feature_placeholders)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Create estimator to train and evaluate\n",
    "def train_and_evaluate(args):\n",
    "  # Create our custome estimator using our model function\n",
    "  estimator = tf.estimator.Estimator(\n",
    "    model_fn = getattr(sys.modules[__name__], args[\"model_fn_name\"]), # finds within current module the model function based on passed string\n",
    "    model_dir = args['output_dir'],\n",
    "    params = {\n",
    "      \"batch_size\": args['batch_size'], \n",
    "      \"input_sequence_length\": args[\"input_sequence_length\"], \n",
    "      \"output_sequence_length\": args[\"output_sequence_length\"],\n",
    "      \"lstm_hidden_units\": args[\"lstm_hidden_units\"],\n",
    "      \"lstm_dropout_output_keep_probs\": args[\"lstm_dropout_output_keep_probs\"], \n",
    "      \"dnn_hidden_units\": args[\"dnn_hidden_units\"], \n",
    "      \"learning_rate\": args['learning_rate']})\n",
    "  # Create train spec to read in our training data\n",
    "  train_spec = tf.estimator.TrainSpec(\n",
    "    input_fn = read_dataset(\n",
    "      filename = args['train_file_pattern'], \n",
    "      mode = tf.estimator.ModeKeys.TRAIN, \n",
    "      batch_size = args['batch_size'] + args['input_sequence_length'] + args['horizon'] + args['output_sequence_length'] - 1,\n",
    "      params = args),\n",
    "    max_steps = args['train_steps'])\n",
    "  # Create exporter to save out the complete model to disk\n",
    "  exporter = tf.estimator.LatestExporter(\n",
    "    name = 'exporter', \n",
    "    serving_input_receiver_fn = lambda: serving_input_fn(input_sequence_length = args[\"input_sequence_length\"], reverse_sequence = args[\"reverse_sequence\"]))\n",
    "  # Create eval spec to read in our validation data and export our model\n",
    "  eval_spec = tf.estimator.EvalSpec(\n",
    "    input_fn = read_dataset(\n",
    "      filename = args['eval_file_pattern'], \n",
    "      mode = tf.estimator.ModeKeys.EVAL, \n",
    "      batch_size = args['batch_size'] + args['input_sequence_length'] + args['horizon'] + args['output_sequence_length'] - 1,\n",
    "      params = args),\n",
    "    steps = None,\n",
    "    start_delay_secs = args[\"start_delay_secs\"], # start evaluating after N seconds\n",
    "    throttle_secs = args[\"throttle_secs\"],  # evaluate every N seconds\n",
    "    exporters = exporter)\n",
    "  # Create train and evaluate loop to train and evaluate our estimator\n",
    "  tf.estimator.train_and_evaluate(estimator = estimator, train_spec = train_spec, eval_spec = eval_spec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_train_distribute': None, '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f4bc17f7b50>, '_evaluation_master': '', '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': 'trained_model', '_global_id_in_cluster': 0, '_save_summary_steps': 100}\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after 30 secs (eval_spec.throttle_secs) or training is finished.\n",
      "\n",
      "read_dataset: _input_fn: filename = ../preproc/price_data_train.csv\n",
      "read_dataset: _input_fn: mode = train\n",
      "read_dataset: _input_fn: batch_size = 10\n",
      "read_dataset: _input_fn: params = {'throttle_secs': 30, 'eval_file_pattern': '../preproc/price_data_eval.csv', 'learning_rate': 0.01, 'dnn_hidden_units': [1024, 256, 64], 'start_delay_secs': 60, 'lstm_dropout_output_keep_probs': [0.95, 1.0, 1.0], 'input_sequence_length': 5, 'train_steps': 1000, 'batch_size': 2, 'horizon': 0, 'reverse_sequence': False, 'model_fn_name': 'encoder_decoder_stacked_lstm_regression', 'train_file_pattern': '../preproc/price_data_train.csv', 'lstm_hidden_units': [64, 32, 16], 'output_sequence_length': 4, 'output_dir': 'trained_model'}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = ['../preproc/price_data_train.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = <TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = Tensor(\"arg0:0\", shape=(), dtype=string)\n",
      "read_dataset: _input_fn: decode_csv: columns = [<tf.Tensor 'DecodeCSV:0' shape=() dtype=float32>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=float32>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=float32>]\n",
      "read_dataset: _input_fn: decode_csv: features = {'dayofweek': <tf.Tensor 'DecodeCSV:1' shape=() dtype=float32>, 'hourofday': <tf.Tensor 'DecodeCSV:2' shape=() dtype=float32>, 'price': <tf.Tensor 'DecodeCSV:0' shape=() dtype=float32>}\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = <MapDataset shapes: {dayofweek: (), hourofday: (), price: ()}, types: {dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = <RepeatDataset shapes: {dayofweek: (), hourofday: (), price: ()}, types: {dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = <_RestructuredDataset shapes: {dayofweek: (10,), hourofday: (10,), price: (10,)}, types: {dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}>\n",
      "\n",
      "read_dataset: _input_fn: create_sequences: features = {'dayofweek': <tf.Tensor 'arg0:0' shape=(10,) dtype=float32>, 'hourofday': <tf.Tensor 'arg1:0' shape=(10,) dtype=float32>, 'price': <tf.Tensor 'arg2:0' shape=(10,) dtype=float32>}\n",
      "read_dataset: _input_fn: create_sequences: sequence_lengths = Tensor(\"Tile:0\", shape=(2,), dtype=int32)\n",
      "read_dataset: _input_fn: create_sequences: label = Tensor(\"strided_slice:0\", shape=(5,), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: label = Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: features[dayofweek] = Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: features[hourofday] = Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: features[price] = Tensor(\"map_3/TensorArrayStack/TensorArrayGatherV3:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: dataset.map(create_sequences) = <MapDataset shapes: ({dayofweek: (2, ?), hourofday: (2, ?), price: (2, ?)}, (2, ?)), types: ({dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}, tf.float32)>\n",
      "read_dataset: _input_fn: dataset.shuffle(buffer_size = 10 * batch_size) = <ShuffleDataset shapes: ({dayofweek: (2, ?), hourofday: (2, ?), price: (2, ?)}, (2, ?)), types: ({dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}, tf.float32)>\n",
      "read_dataset: _input_fn: batch_features = {'dayofweek': <tf.Tensor 'IteratorGetNext:0' shape=(2, ?) dtype=float32>, 'hourofday': <tf.Tensor 'IteratorGetNext:1' shape=(2, ?) dtype=float32>, 'price': <tf.Tensor 'IteratorGetNext:2' shape=(2, ?) dtype=float32>}\n",
      "read_dataset: _input_fn: batch_labels = Tensor(\"IteratorGetNext:3\", shape=(2, ?), dtype=float32, device=/device:CPU:0)\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: features = {'dayofweek': <tf.Tensor 'IteratorGetNext:0' shape=(2, ?) dtype=float32>, 'hourofday': <tf.Tensor 'IteratorGetNext:1' shape=(2, ?) dtype=float32>, 'price': <tf.Tensor 'IteratorGetNext:2' shape=(2, ?) dtype=float32>}\n",
      "encoder_decoder_stacked_lstm_regression: labels = Tensor(\"IteratorGetNext:3\", shape=(2, ?), dtype=float32, device=/device:CPU:0)\n",
      "encoder_decoder_stacked_lstm_regression: mode = train\n",
      "encoder_decoder_stacked_lstm_regression: params = {'learning_rate': 0.01, 'lstm_dropout_output_keep_probs': [0.95, 1.0, 1.0], 'input_sequence_length': 5, 'batch_size': 2, 'dnn_hidden_units': [1024, 256, 64], 'output_sequence_length': 4, 'lstm_hidden_units': [64, 32, 16]}\n",
      "encoder_decoder_stacked_lstm_regression: current_batch_size = Tensor(\"strided_slice:0\", shape=(), dtype=int32)\n",
      "encoder_decoder_stacked_lstm_regression: number_of_features = 3\n",
      "encoder_decoder_stacked_lstm_regression: X = Tensor(\"stack:0\", shape=(2, ?, 3), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: X_sequence = [<tf.Tensor 'unstack:0' shape=(2, 3) dtype=float32>, <tf.Tensor 'unstack:1' shape=(2, 3) dtype=float32>, <tf.Tensor 'unstack:2' shape=(2, 3) dtype=float32>, <tf.Tensor 'unstack:3' shape=(2, 3) dtype=float32>, <tf.Tensor 'unstack:4' shape=(2, 3) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: lstm_cells = [<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f4bd102ca50>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f4bc3aa4610>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f4bc3aa4710>]\n",
      "encoder_decoder_stacked_lstm_regression: dropout_lstm_cells = [<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x7f4bc3aa4790>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x7f4bc3aa47d0>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x7f4bc3aa4850>]\n",
      "encoder_decoder_stacked_lstm_regression: stacked_lstm_cells = <tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x7f4bd1009fd0>\n",
      "encoder_decoder_stacked_lstm_regression: encoder_cells = <tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x7f4bc3ac3350>\n",
      "encoder_decoder_stacked_lstm_regression: encoder_outputs = [<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_14:0' shape=(2, 16) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: encoder_final_state = (LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Add_9:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_14:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Add_9:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_14:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Add_9:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_14:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: unstacked_labels = [<tf.Tensor 'unstack_1:0' shape=(2,) dtype=float32>, <tf.Tensor 'unstack_1:1' shape=(2,) dtype=float32>, <tf.Tensor 'unstack_1:2' shape=(2,) dtype=float32>, <tf.Tensor 'unstack_1:3' shape=(2,) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: expanded_unstacked_labels = [<tf.Tensor 'ExpandDims:0' shape=(2, 1) dtype=float32>, <tf.Tensor 'ExpandDims_1:0' shape=(2, 1) dtype=float32>, <tf.Tensor 'ExpandDims_2:0' shape=(2, 1) dtype=float32>, <tf.Tensor 'ExpandDims_3:0' shape=(2, 1) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Add_9:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_14:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Add_9:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_14:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Add_9:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_14:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_1:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_2:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_1:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_2:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_1:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn/dense/Relu:0\", shape=(2, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn/dense_1/Relu:0\", shape=(2, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn/dense_2/Relu:0\", shape=(2, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: logits = Tensor(\"decoder/dnn/dense_3/BiasAdd:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: decoder_input = Tensor(\"ExpandDims_1:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_3:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_5:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_3:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_5:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_3:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_1/dense/Relu:0\", shape=(2, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_1/dense_1/Relu:0\", shape=(2, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_1/dense_2/Relu:0\", shape=(2, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: logits = Tensor(\"decoder/dnn_1/dense_3/BiasAdd:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: decoder_input = Tensor(\"ExpandDims_2:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_5:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_8:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_5:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_8:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_5:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_2/dense/Relu:0\", shape=(2, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_2/dense_1/Relu:0\", shape=(2, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_2/dense_2/Relu:0\", shape=(2, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: logits = Tensor(\"decoder/dnn_2/dense_3/BiasAdd:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: decoder_input = Tensor(\"ExpandDims_3:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_7:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_11:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_7:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_11:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_7:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: decoder_outputs = [<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(2, 16) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: decoder_states = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_7:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_11:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_7:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_11:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_7:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: stacked_decoder_outputs = Tensor(\"stack_1:0\", shape=(4, 2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"stack_1:0\", shape=(4, 2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"dnn/dense/Relu:0\", shape=(4, 2, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"dnn/dense_1/Relu:0\", shape=(4, 2, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"dnn/dense_2/Relu:0\", shape=(4, 2, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: logits = Tensor(\"dnn/dense_3/BiasAdd:0\", shape=(4, 2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: predictions = Tensor(\"Reshape:0\", shape=(2, 4), dtype=float32)\n",
      "\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into trained_model/model.ckpt.\n",
      "INFO:tensorflow:loss = 1796.878, step = 1\n",
      "INFO:tensorflow:global_step/sec: 42.0798\n",
      "INFO:tensorflow:loss = 5.821585, step = 101 (2.384 sec)\n",
      "INFO:tensorflow:global_step/sec: 116.742\n",
      "INFO:tensorflow:loss = 257.75156, step = 201 (0.857 sec)\n",
      "INFO:tensorflow:global_step/sec: 111.525\n",
      "INFO:tensorflow:loss = 78.893814, step = 301 (0.896 sec)\n",
      "INFO:tensorflow:global_step/sec: 107.967\n",
      "INFO:tensorflow:loss = 35.79174, step = 401 (0.926 sec)\n",
      "INFO:tensorflow:global_step/sec: 110.608\n",
      "INFO:tensorflow:loss = 6.454268, step = 501 (0.912 sec)\n",
      "INFO:tensorflow:global_step/sec: 113.028\n",
      "INFO:tensorflow:loss = 1.5590835, step = 601 (0.880 sec)\n",
      "INFO:tensorflow:global_step/sec: 115.875\n",
      "INFO:tensorflow:loss = 3.5502214, step = 701 (0.860 sec)\n",
      "INFO:tensorflow:global_step/sec: 116.898\n",
      "INFO:tensorflow:loss = 84.54157, step = 801 (0.856 sec)\n",
      "INFO:tensorflow:global_step/sec: 115.535\n",
      "INFO:tensorflow:loss = 18.075583, step = 901 (0.865 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1000 into trained_model/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 3.2752147.\n",
      "\n",
      "read_dataset: _input_fn: filename = ../preproc/price_data_eval.csv\n",
      "read_dataset: _input_fn: mode = eval\n",
      "read_dataset: _input_fn: batch_size = 10\n",
      "read_dataset: _input_fn: params = {'throttle_secs': 30, 'eval_file_pattern': '../preproc/price_data_eval.csv', 'learning_rate': 0.01, 'dnn_hidden_units': [1024, 256, 64], 'start_delay_secs': 60, 'lstm_dropout_output_keep_probs': [0.95, 1.0, 1.0], 'input_sequence_length': 5, 'train_steps': 1000, 'batch_size': 2, 'horizon': 0, 'reverse_sequence': False, 'model_fn_name': 'encoder_decoder_stacked_lstm_regression', 'train_file_pattern': '../preproc/price_data_train.csv', 'lstm_hidden_units': [64, 32, 16], 'output_sequence_length': 4, 'output_dir': 'trained_model'}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = ['../preproc/price_data_eval.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = <TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = Tensor(\"arg0:0\", shape=(), dtype=string)\n",
      "read_dataset: _input_fn: decode_csv: columns = [<tf.Tensor 'DecodeCSV:0' shape=() dtype=float32>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=float32>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=float32>]\n",
      "read_dataset: _input_fn: decode_csv: features = {'dayofweek': <tf.Tensor 'DecodeCSV:1' shape=() dtype=float32>, 'hourofday': <tf.Tensor 'DecodeCSV:2' shape=() dtype=float32>, 'price': <tf.Tensor 'DecodeCSV:0' shape=() dtype=float32>}\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = <MapDataset shapes: {dayofweek: (), hourofday: (), price: ()}, types: {dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = <RepeatDataset shapes: {dayofweek: (), hourofday: (), price: ()}, types: {dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = <_RestructuredDataset shapes: {dayofweek: (10,), hourofday: (10,), price: (10,)}, types: {dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}>\n",
      "\n",
      "read_dataset: _input_fn: create_sequences: features = {'dayofweek': <tf.Tensor 'arg0:0' shape=(10,) dtype=float32>, 'hourofday': <tf.Tensor 'arg1:0' shape=(10,) dtype=float32>, 'price': <tf.Tensor 'arg2:0' shape=(10,) dtype=float32>}\n",
      "read_dataset: _input_fn: create_sequences: sequence_lengths = Tensor(\"Tile:0\", shape=(2,), dtype=int32)\n",
      "read_dataset: _input_fn: create_sequences: label = Tensor(\"strided_slice:0\", shape=(5,), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: label = Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: features[dayofweek] = Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: features[hourofday] = Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: features[price] = Tensor(\"map_3/TensorArrayStack/TensorArrayGatherV3:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: dataset.map(create_sequences) = <MapDataset shapes: ({dayofweek: (2, ?), hourofday: (2, ?), price: (2, ?)}, (2, ?)), types: ({dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}, tf.float32)>\n",
      "read_dataset: _input_fn: batch_features = {'dayofweek': <tf.Tensor 'IteratorGetNext:0' shape=(2, ?) dtype=float32>, 'hourofday': <tf.Tensor 'IteratorGetNext:1' shape=(2, ?) dtype=float32>, 'price': <tf.Tensor 'IteratorGetNext:2' shape=(2, ?) dtype=float32>}\n",
      "read_dataset: _input_fn: batch_labels = Tensor(\"IteratorGetNext:3\", shape=(2, ?), dtype=float32, device=/device:CPU:0)\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: features = {'dayofweek': <tf.Tensor 'IteratorGetNext:0' shape=(2, ?) dtype=float32>, 'hourofday': <tf.Tensor 'IteratorGetNext:1' shape=(2, ?) dtype=float32>, 'price': <tf.Tensor 'IteratorGetNext:2' shape=(2, ?) dtype=float32>}\n",
      "encoder_decoder_stacked_lstm_regression: labels = Tensor(\"IteratorGetNext:3\", shape=(2, ?), dtype=float32, device=/device:CPU:0)\n",
      "encoder_decoder_stacked_lstm_regression: mode = eval\n",
      "encoder_decoder_stacked_lstm_regression: params = {'learning_rate': 0.01, 'lstm_dropout_output_keep_probs': [0.95, 1.0, 1.0], 'input_sequence_length': 5, 'batch_size': 2, 'dnn_hidden_units': [1024, 256, 64], 'output_sequence_length': 4, 'lstm_hidden_units': [64, 32, 16]}\n",
      "encoder_decoder_stacked_lstm_regression: current_batch_size = Tensor(\"strided_slice:0\", shape=(), dtype=int32)\n",
      "encoder_decoder_stacked_lstm_regression: number_of_features = 3\n",
      "encoder_decoder_stacked_lstm_regression: X = Tensor(\"stack:0\", shape=(2, ?, 3), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: X_sequence = [<tf.Tensor 'unstack:0' shape=(2, 3) dtype=float32>, <tf.Tensor 'unstack:1' shape=(2, 3) dtype=float32>, <tf.Tensor 'unstack:2' shape=(2, 3) dtype=float32>, <tf.Tensor 'unstack:3' shape=(2, 3) dtype=float32>, <tf.Tensor 'unstack:4' shape=(2, 3) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: lstm_cells = [<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f4bd0f03e90>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f4bd0f2e4d0>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f4bd0f2e5d0>]\n",
      "encoder_decoder_stacked_lstm_regression: dropout_lstm_cells = [<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x7f4bd0f2e650>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x7f4bd0f2e690>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x7f4bd0f2e710>]\n",
      "encoder_decoder_stacked_lstm_regression: stacked_lstm_cells = <tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x7f4bc265dbd0>\n",
      "encoder_decoder_stacked_lstm_regression: encoder_cells = <tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x7f4bd0e9e210>\n",
      "encoder_decoder_stacked_lstm_regression: encoder_outputs = [<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_14:0' shape=(2, 16) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: encoder_final_state = (LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Add_9:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_14:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Add_9:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_14:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Add_9:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_14:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: unstacked_labels = [<tf.Tensor 'unstack_1:0' shape=(2,) dtype=float32>, <tf.Tensor 'unstack_1:1' shape=(2,) dtype=float32>, <tf.Tensor 'unstack_1:2' shape=(2,) dtype=float32>, <tf.Tensor 'unstack_1:3' shape=(2,) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: expanded_unstacked_labels = [<tf.Tensor 'ExpandDims:0' shape=(2, 1) dtype=float32>, <tf.Tensor 'ExpandDims_1:0' shape=(2, 1) dtype=float32>, <tf.Tensor 'ExpandDims_2:0' shape=(2, 1) dtype=float32>, <tf.Tensor 'ExpandDims_3:0' shape=(2, 1) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Add_9:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_14:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Add_9:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_14:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Add_9:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_14:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_1:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_2:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_1:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_2:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_1:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn/dense/Relu:0\", shape=(2, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn/dense_1/Relu:0\", shape=(2, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn/dense_2/Relu:0\", shape=(2, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: logits = Tensor(\"decoder/dnn/dense_3/BiasAdd:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: decoder_input = Tensor(\"ExpandDims_1:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_3:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_5:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_3:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_5:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_3:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_1/dense/Relu:0\", shape=(2, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_1/dense_1/Relu:0\", shape=(2, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_1/dense_2/Relu:0\", shape=(2, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: logits = Tensor(\"decoder/dnn_1/dense_3/BiasAdd:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: decoder_input = Tensor(\"ExpandDims_2:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_5:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_8:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_5:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_8:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_5:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_2/dense/Relu:0\", shape=(2, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_2/dense_1/Relu:0\", shape=(2, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_2/dense_2/Relu:0\", shape=(2, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: logits = Tensor(\"decoder/dnn_2/dense_3/BiasAdd:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: decoder_input = Tensor(\"ExpandDims_3:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_7:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_11:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_7:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_11:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_7:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: decoder_outputs = [<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(2, 16) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: decoder_states = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_7:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_11:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_7:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_11:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_7:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: stacked_decoder_outputs = Tensor(\"stack_1:0\", shape=(4, 2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"stack_1:0\", shape=(4, 2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"dnn/dense/Relu:0\", shape=(4, 2, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"dnn/dense_1/Relu:0\", shape=(4, 2, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"dnn/dense_2/Relu:0\", shape=(4, 2, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: logits = Tensor(\"dnn/dense_3/BiasAdd:0\", shape=(4, 2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: predictions = Tensor(\"Reshape:0\", shape=(2, 4), dtype=float32)\n",
      "\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2018-05-25-05:49:38\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-1000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2018-05-25-05:49:39\n",
      "INFO:tensorflow:Saving dict for global step 1000: global_step = 1000, loss = 19.524153, mae = 3.5430794, rmse = 4.4186144\n",
      "\n",
      "serving_input_fn: feature_placeholders = {'dayofweek': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>, 'hourofday': <tf.Tensor 'Placeholder_2:0' shape=(?,) dtype=string>, 'price': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>}\n",
      "serving_input_fn: fix_shape_and_type_for_serving: split_string = Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=string)\n",
      "serving_input_fn: fix_shape_and_type_for_serving: feature_tensor = Tensor(\"StringToNumber:0\", shape=(?, ?), dtype=float32)\n",
      "serving_input_fn: fix_shape_and_type_for_serving: split_string = Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=string)\n",
      "serving_input_fn: fix_shape_and_type_for_serving: feature_tensor = Tensor(\"StringToNumber_1:0\", shape=(?, ?), dtype=float32)\n",
      "serving_input_fn: fix_shape_and_type_for_serving: split_string = Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=string)\n",
      "serving_input_fn: fix_shape_and_type_for_serving: feature_tensor = Tensor(\"StringToNumber_2:0\", shape=(?, ?), dtype=float32)\n",
      "serving_input_fn: features = {'dayofweek': <tf.Tensor 'StringToNumber:0' shape=(?, ?) dtype=float32>, 'hourofday': <tf.Tensor 'StringToNumber_1:0' shape=(?, ?) dtype=float32>, 'price': <tf.Tensor 'StringToNumber_2:0' shape=(?, ?) dtype=float32>}\n",
      "serving_input_fn: get_shape_and_set_modified_shape_2D: tensor = Tensor(\"StringToNumber:0\", shape=(?, 5), dtype=float32), additional_dimension_sizes = [5]\n",
      "serving_input_fn: get_shape_and_set_modified_shape_2D: tensor = Tensor(\"StringToNumber_1:0\", shape=(?, 5), dtype=float32), additional_dimension_sizes = [5]\n",
      "serving_input_fn: get_shape_and_set_modified_shape_2D: tensor = Tensor(\"StringToNumber_2:0\", shape=(?, 5), dtype=float32), additional_dimension_sizes = [5]\n",
      "serving_input_fn: features = {'dayofweek': <tf.Tensor 'StringToNumber:0' shape=(?, 5) dtype=float32>, 'hourofday': <tf.Tensor 'StringToNumber_1:0' shape=(?, 5) dtype=float32>, 'price': <tf.Tensor 'StringToNumber_2:0' shape=(?, 5) dtype=float32>}\n",
      "serving_input_fn: sequence_lengths = Tensor(\"Tile:0\", shape=(?,), dtype=int32)\n",
      "serving_input_fn: features = {'dayofweek': <tf.Tensor 'StringToNumber:0' shape=(?, 5) dtype=float32>, 'hourofday': <tf.Tensor 'StringToNumber_1:0' shape=(?, 5) dtype=float32>, 'price': <tf.Tensor 'StringToNumber_2:0' shape=(?, 5) dtype=float32>}\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: features = {'dayofweek': <tf.Tensor 'StringToNumber:0' shape=(?, 5) dtype=float32>, 'hourofday': <tf.Tensor 'StringToNumber_1:0' shape=(?, 5) dtype=float32>, 'price': <tf.Tensor 'StringToNumber_2:0' shape=(?, 5) dtype=float32>}\n",
      "encoder_decoder_stacked_lstm_regression: labels = None\n",
      "encoder_decoder_stacked_lstm_regression: mode = infer\n",
      "encoder_decoder_stacked_lstm_regression: params = {'learning_rate': 0.01, 'lstm_dropout_output_keep_probs': [0.95, 1.0, 1.0], 'input_sequence_length': 5, 'batch_size': 2, 'dnn_hidden_units': [1024, 256, 64], 'output_sequence_length': 4, 'lstm_hidden_units': [64, 32, 16]}\n",
      "encoder_decoder_stacked_lstm_regression: current_batch_size = Tensor(\"strided_slice_4:0\", shape=(), dtype=int32)\n",
      "encoder_decoder_stacked_lstm_regression: number_of_features = 3\n",
      "encoder_decoder_stacked_lstm_regression: X = Tensor(\"stack:0\", shape=(?, 5, 3), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: X_sequence = [<tf.Tensor 'unstack:0' shape=(?, 3) dtype=float32>, <tf.Tensor 'unstack:1' shape=(?, 3) dtype=float32>, <tf.Tensor 'unstack:2' shape=(?, 3) dtype=float32>, <tf.Tensor 'unstack:3' shape=(?, 3) dtype=float32>, <tf.Tensor 'unstack:4' shape=(?, 3) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: lstm_cells = [<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f4bc2a45f50>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f4bd13f6810>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f4bd13f6910>]\n",
      "encoder_decoder_stacked_lstm_regression: dropout_lstm_cells = [<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x7f4bd13f6990>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x7f4bd13f69d0>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x7f4bd13f6a50>]\n",
      "encoder_decoder_stacked_lstm_regression: stacked_lstm_cells = <tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x7f4bc2a45710>\n",
      "encoder_decoder_stacked_lstm_regression: encoder_cells = <tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x7f4bd13ea550>\n",
      "encoder_decoder_stacked_lstm_regression: encoder_outputs = [<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0' shape=(?, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0' shape=(?, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0' shape=(?, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(?, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_14:0' shape=(?, 16) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: encoder_final_state = (LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Add_9:0' shape=(?, 64) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_14:0' shape=(?, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Add_9:0' shape=(?, 32) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_14:0' shape=(?, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Add_9:0' shape=(?, 16) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_14:0' shape=(?, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: fake_labels = [<tf.Tensor 'zeros:0' shape=(?, 1) dtype=float32>, <tf.Tensor 'zeros_1:0' shape=(?, 1) dtype=float32>, <tf.Tensor 'zeros_2:0' shape=(?, 1) dtype=float32>, <tf.Tensor 'zeros_3:0' shape=(?, 1) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Add_9:0' shape=(?, 64) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_14:0' shape=(?, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Add_9:0' shape=(?, 32) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_14:0' shape=(?, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Add_9:0' shape=(?, 16) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_14:0' shape=(?, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0\", shape=(?, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_1:0' shape=(?, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_2:0' shape=(?, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_1:0' shape=(?, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_2:0' shape=(?, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_1:0' shape=(?, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0' shape=(?, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0\", shape=(?, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn/dense/Relu:0\", shape=(?, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn/dense_1/Relu:0\", shape=(?, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn/dense_2/Relu:0\", shape=(?, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: logits = Tensor(\"decoder/dnn/dense_3/BiasAdd:0\", shape=(?, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: decoder_input = Tensor(\"decoder/dnn/dense_3/BiasAdd:0\", shape=(?, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0\", shape=(?, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_3:0' shape=(?, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_5:0' shape=(?, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_3:0' shape=(?, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_5:0' shape=(?, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_3:0' shape=(?, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0' shape=(?, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0\", shape=(?, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_1/dense/Relu:0\", shape=(?, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_1/dense_1/Relu:0\", shape=(?, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_1/dense_2/Relu:0\", shape=(?, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: logits = Tensor(\"decoder/dnn_1/dense_3/BiasAdd:0\", shape=(?, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: decoder_input = Tensor(\"decoder/dnn_1/dense_3/BiasAdd:0\", shape=(?, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0\", shape=(?, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_5:0' shape=(?, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_8:0' shape=(?, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_5:0' shape=(?, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_8:0' shape=(?, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_5:0' shape=(?, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0' shape=(?, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0\", shape=(?, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_2/dense/Relu:0\", shape=(?, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_2/dense_1/Relu:0\", shape=(?, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_2/dense_2/Relu:0\", shape=(?, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: logits = Tensor(\"decoder/dnn_2/dense_3/BiasAdd:0\", shape=(?, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: decoder_input = Tensor(\"decoder/dnn_2/dense_3/BiasAdd:0\", shape=(?, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0\", shape=(?, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_7:0' shape=(?, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_11:0' shape=(?, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_7:0' shape=(?, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_11:0' shape=(?, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_7:0' shape=(?, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(?, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: decoder_outputs = [<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0' shape=(?, 16) dtype=float32>, <tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0' shape=(?, 16) dtype=float32>, <tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0' shape=(?, 16) dtype=float32>, <tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(?, 16) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: decoder_states = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_7:0' shape=(?, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_11:0' shape=(?, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_7:0' shape=(?, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_11:0' shape=(?, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_7:0' shape=(?, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(?, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: stacked_decoder_outputs = Tensor(\"stack_1:0\", shape=(4, ?, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"stack_1:0\", shape=(4, ?, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"dnn/dense/Relu:0\", shape=(4, ?, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"dnn/dense_1/Relu:0\", shape=(4, ?, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"dnn/dense_2/Relu:0\", shape=(4, ?, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: logits = Tensor(\"dnn/dense_3/BiasAdd:0\", shape=(4, ?, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: predictions = Tensor(\"Reshape:0\", shape=(?, 4), dtype=float32)\n",
      "\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['serving_default', 'predict_export_outputs']\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-1000\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: trained_model/export/exporter/temp-1527227380/saved_model.pb\n"
     ]
    }
   ],
   "source": [
    "# Run the model\n",
    "shutil.rmtree(arguments[\"output_dir\"], ignore_errors = True) # start fresh each time\n",
    "train_and_evaluate(arguments)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Locally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Will use LSTM size of [64, 32, 16]\n",
      "Will use LSTM output keep probabilities of [0.9, 1.0, 1.0]\n",
      "Will use DNN size of [1024, 256, 64]\n",
      "\n",
      "read_dataset: _input_fn: filename = ../preproc/price_data_train.csv\n",
      "read_dataset: _input_fn: mode = train\n",
      "read_dataset: _input_fn: batch_size = 10\n",
      "read_dataset: _input_fn: params = {'throttle_secs': 30, 'eval_file_pattern': '../preproc/price_data_eval.csv', 'input_sequence_length': 5, 'lstm_hidden_units': [64, 32, 16], 'learning_rate': 0.01, 'dnn_hidden_units': [1024, 256, 64], 'lstm_dropout_output_keep_probs': [0.9, 1.0, 1.0], 'batch_size': 2, 'output_dir': '/content/datalab/notebooks/05-17-18/youtube-8m-team/ryan_energy_models/meta_models/trained_model/', 'horizon': 0, 'reverse_sequence': True, 'model_fn_name': 'encoder_decoder_stacked_lstm_regression', 'train_file_pattern': '../preproc/price_data_train.csv', 'output_sequence_length': 4, 'train_steps': 1000, 'start_delay_secs': 60}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = ['../preproc/price_data_train.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = <TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = Tensor(\"arg0:0\", shape=(), dtype=string)\n",
      "read_dataset: _input_fn: decode_csv: columns = [<tf.Tensor 'DecodeCSV:0' shape=() dtype=float32>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=float32>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=float32>]\n",
      "read_dataset: _input_fn: decode_csv: features = {'dayofweek': <tf.Tensor 'DecodeCSV:1' shape=() dtype=float32>, 'hourofday': <tf.Tensor 'DecodeCSV:2' shape=() dtype=float32>, 'price': <tf.Tensor 'DecodeCSV:0' shape=() dtype=float32>}\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = <MapDataset shapes: {dayofweek: (), hourofday: (), price: ()}, types: {dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = <RepeatDataset shapes: {dayofweek: (), hourofday: (), price: ()}, types: {dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = <_RestructuredDataset shapes: {dayofweek: (10,), hourofday: (10,), price: (10,)}, types: {dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}>\n",
      "\n",
      "read_dataset: _input_fn: create_sequences: features = {'dayofweek': <tf.Tensor 'arg0:0' shape=(10,) dtype=float32>, 'hourofday': <tf.Tensor 'arg1:0' shape=(10,) dtype=float32>, 'price': <tf.Tensor 'arg2:0' shape=(10,) dtype=float32>}\n",
      "read_dataset: _input_fn: create_sequences: sequence_lengths = Tensor(\"Tile:0\", shape=(2,), dtype=int32)\n",
      "read_dataset: _input_fn: create_sequences: label = Tensor(\"strided_slice:0\", shape=(5,), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: label = Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: features[dayofweek] = Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: reversed features[dayofweek] = Tensor(\"ReverseSequence:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: features[hourofday] = Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: reversed features[hourofday] = Tensor(\"ReverseSequence_1:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: features[price] = Tensor(\"map_3/TensorArrayStack/TensorArrayGatherV3:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: reversed features[price] = Tensor(\"ReverseSequence_2:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: dataset.map(create_sequences) = <MapDataset shapes: ({dayofweek: (2, ?), hourofday: (2, ?), price: (2, ?)}, (2, ?)), types: ({dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}, tf.float32)>\n",
      "read_dataset: _input_fn: dataset.shuffle(buffer_size = 10 * batch_size) = <ShuffleDataset shapes: ({dayofweek: (2, ?), hourofday: (2, ?), price: (2, ?)}, (2, ?)), types: ({dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}, tf.float32)>\n",
      "read_dataset: _input_fn: batch_features = {'dayofweek': <tf.Tensor 'IteratorGetNext:0' shape=(2, ?) dtype=float32>, 'hourofday': <tf.Tensor 'IteratorGetNext:1' shape=(2, ?) dtype=float32>, 'price': <tf.Tensor 'IteratorGetNext:2' shape=(2, ?) dtype=float32>}\n",
      "read_dataset: _input_fn: batch_labels = Tensor(\"IteratorGetNext:3\", shape=(2, ?), dtype=float32, device=/device:CPU:0)\n",
      "\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: features = {'dayofweek': <tf.Tensor 'IteratorGetNext:0' shape=(2, ?) dtype=float32>, 'hourofday': <tf.Tensor 'IteratorGetNext:1' shape=(2, ?) dtype=float32>, 'price': <tf.Tensor 'IteratorGetNext:2' shape=(2, ?) dtype=float32>}\n",
      "encoder_decoder_stacked_lstm_regression: labels = Tensor(\"IteratorGetNext:3\", shape=(2, ?), dtype=float32, device=/device:CPU:0)\n",
      "encoder_decoder_stacked_lstm_regression: mode = train\n",
      "encoder_decoder_stacked_lstm_regression: params = {'learning_rate': 0.01, 'lstm_dropout_output_keep_probs': [0.9, 1.0, 1.0], 'input_sequence_length': 5, 'batch_size': 2, 'dnn_hidden_units': [1024, 256, 64], 'output_sequence_length': 4, 'lstm_hidden_units': [64, 32, 16]}\n",
      "encoder_decoder_stacked_lstm_regression: current_batch_size = Tensor(\"strided_slice:0\", shape=(), dtype=int32)\n",
      "encoder_decoder_stacked_lstm_regression: number_of_features = 3\n",
      "encoder_decoder_stacked_lstm_regression: X = Tensor(\"stack:0\", shape=(2, ?, 3), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: X_sequence = [<tf.Tensor 'unstack:0' shape=(2, 3) dtype=float32>, <tf.Tensor 'unstack:1' shape=(2, 3) dtype=float32>, <tf.Tensor 'unstack:2' shape=(2, 3) dtype=float32>, <tf.Tensor 'unstack:3' shape=(2, 3) dtype=float32>, <tf.Tensor 'unstack:4' shape=(2, 3) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: lstm_cells = [<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7faf370b6890>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7faf10b04490>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7faf10b04590>]\n",
      "encoder_decoder_stacked_lstm_regression: dropout_lstm_cells = [<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x7faf10b04610>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x7faf10b04650>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x7faf10b046d0>]\n",
      "encoder_decoder_stacked_lstm_regression: stacked_lstm_cells = <tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x7faf10b5b490>\n",
      "encoder_decoder_stacked_lstm_regression: encoder_cells = <tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x7faf10b111d0>\n",
      "encoder_decoder_stacked_lstm_regression: encoder_outputs = [<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_14:0' shape=(2, 16) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: encoder_final_state = (LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Add_9:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_14:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Add_9:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_14:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Add_9:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_14:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: unstacked_labels = [<tf.Tensor 'unstack_1:0' shape=(2,) dtype=float32>, <tf.Tensor 'unstack_1:1' shape=(2,) dtype=float32>, <tf.Tensor 'unstack_1:2' shape=(2,) dtype=float32>, <tf.Tensor 'unstack_1:3' shape=(2,) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: expanded_unstacked_labels = [<tf.Tensor 'ExpandDims:0' shape=(2, 1) dtype=float32>, <tf.Tensor 'ExpandDims_1:0' shape=(2, 1) dtype=float32>, <tf.Tensor 'ExpandDims_2:0' shape=(2, 1) dtype=float32>, <tf.Tensor 'ExpandDims_3:0' shape=(2, 1) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Add_9:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_14:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Add_9:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_14:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Add_9:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_14:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_1:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_2:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_1:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_2:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_1:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn/dense/Relu:0\", shape=(2, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn/dense_1/Relu:0\", shape=(2, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn/dense_2/Relu:0\", shape=(2, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: logits = Tensor(\"decoder/dnn/dense_3/BiasAdd:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: decoder_input = Tensor(\"ExpandDims_1:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_3:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_5:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_3:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_5:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_3:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_1/dense/Relu:0\", shape=(2, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_1/dense_1/Relu:0\", shape=(2, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_1/dense_2/Relu:0\", shape=(2, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: logits = Tensor(\"decoder/dnn_1/dense_3/BiasAdd:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: decoder_input = Tensor(\"ExpandDims_2:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_5:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_8:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_5:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_8:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_5:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_2/dense/Relu:0\", shape=(2, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_2/dense_1/Relu:0\", shape=(2, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_2/dense_2/Relu:0\", shape=(2, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: logits = Tensor(\"decoder/dnn_2/dense_3/BiasAdd:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: decoder_input = Tensor(\"ExpandDims_3:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_7:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_11:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_7:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_11:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_7:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: decoder_outputs = [<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(2, 16) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: decoder_states = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_7:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_11:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_7:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_11:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_7:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: stacked_decoder_outputs = Tensor(\"stack_1:0\", shape=(4, 2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"stack_1:0\", shape=(4, 2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"dnn/dense/Relu:0\", shape=(4, 2, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"dnn/dense_1/Relu:0\", shape=(4, 2, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"dnn/dense_2/Relu:0\", shape=(4, 2, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: logits = Tensor(\"dnn/dense_3/BiasAdd:0\", shape=(4, 2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: predictions = Tensor(\"Reshape:0\", shape=(2, 4), dtype=float32)\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: filename = ../preproc/price_data_eval.csv\n",
      "read_dataset: _input_fn: mode = eval\n",
      "read_dataset: _input_fn: batch_size = 10\n",
      "read_dataset: _input_fn: params = {'throttle_secs': 30, 'eval_file_pattern': '../preproc/price_data_eval.csv', 'input_sequence_length': 5, 'lstm_hidden_units': [64, 32, 16], 'learning_rate': 0.01, 'dnn_hidden_units': [1024, 256, 64], 'lstm_dropout_output_keep_probs': [0.9, 1.0, 1.0], 'batch_size': 2, 'output_dir': '/content/datalab/notebooks/05-17-18/youtube-8m-team/ryan_energy_models/meta_models/trained_model/', 'horizon': 0, 'reverse_sequence': True, 'model_fn_name': 'encoder_decoder_stacked_lstm_regression', 'train_file_pattern': '../preproc/price_data_train.csv', 'output_sequence_length': 4, 'train_steps': 1000, 'start_delay_secs': 60}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = ['../preproc/price_data_eval.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = <TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = Tensor(\"arg0:0\", shape=(), dtype=string)\n",
      "read_dataset: _input_fn: decode_csv: columns = [<tf.Tensor 'DecodeCSV:0' shape=() dtype=float32>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=float32>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=float32>]\n",
      "read_dataset: _input_fn: decode_csv: features = {'dayofweek': <tf.Tensor 'DecodeCSV:1' shape=() dtype=float32>, 'hourofday': <tf.Tensor 'DecodeCSV:2' shape=() dtype=float32>, 'price': <tf.Tensor 'DecodeCSV:0' shape=() dtype=float32>}\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = <MapDataset shapes: {dayofweek: (), hourofday: (), price: ()}, types: {dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = <RepeatDataset shapes: {dayofweek: (), hourofday: (), price: ()}, types: {dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = <_RestructuredDataset shapes: {dayofweek: (10,), hourofday: (10,), price: (10,)}, types: {dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}>\n",
      "\n",
      "read_dataset: _input_fn: create_sequences: features = {'dayofweek': <tf.Tensor 'arg0:0' shape=(10,) dtype=float32>, 'hourofday': <tf.Tensor 'arg1:0' shape=(10,) dtype=float32>, 'price': <tf.Tensor 'arg2:0' shape=(10,) dtype=float32>}\n",
      "read_dataset: _input_fn: create_sequences: sequence_lengths = Tensor(\"Tile:0\", shape=(2,), dtype=int32)\n",
      "read_dataset: _input_fn: create_sequences: label = Tensor(\"strided_slice:0\", shape=(5,), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: label = Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: features[dayofweek] = Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: reversed features[dayofweek] = Tensor(\"ReverseSequence:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: features[hourofday] = Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: reversed features[hourofday] = Tensor(\"ReverseSequence_1:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: features[price] = Tensor(\"map_3/TensorArrayStack/TensorArrayGatherV3:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: create_sequences: reversed features[price] = Tensor(\"ReverseSequence_2:0\", shape=(2, ?), dtype=float32)\n",
      "read_dataset: _input_fn: dataset.map(create_sequences) = <MapDataset shapes: ({dayofweek: (2, ?), hourofday: (2, ?), price: (2, ?)}, (2, ?)), types: ({dayofweek: tf.float32, hourofday: tf.float32, price: tf.float32}, tf.float32)>\n",
      "read_dataset: _input_fn: batch_features = {'dayofweek': <tf.Tensor 'IteratorGetNext:0' shape=(2, ?) dtype=float32>, 'hourofday': <tf.Tensor 'IteratorGetNext:1' shape=(2, ?) dtype=float32>, 'price': <tf.Tensor 'IteratorGetNext:2' shape=(2, ?) dtype=float32>}\n",
      "read_dataset: _input_fn: batch_labels = Tensor(\"IteratorGetNext:3\", shape=(2, ?), dtype=float32, device=/device:CPU:0)\n",
      "\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: features = {'dayofweek': <tf.Tensor 'IteratorGetNext:0' shape=(2, ?) dtype=float32>, 'hourofday': <tf.Tensor 'IteratorGetNext:1' shape=(2, ?) dtype=float32>, 'price': <tf.Tensor 'IteratorGetNext:2' shape=(2, ?) dtype=float32>}\n",
      "encoder_decoder_stacked_lstm_regression: labels = Tensor(\"IteratorGetNext:3\", shape=(2, ?), dtype=float32, device=/device:CPU:0)\n",
      "encoder_decoder_stacked_lstm_regression: mode = eval\n",
      "encoder_decoder_stacked_lstm_regression: params = {'learning_rate': 0.01, 'lstm_dropout_output_keep_probs': [0.9, 1.0, 1.0], 'input_sequence_length': 5, 'batch_size': 2, 'dnn_hidden_units': [1024, 256, 64], 'output_sequence_length': 4, 'lstm_hidden_units': [64, 32, 16]}\n",
      "encoder_decoder_stacked_lstm_regression: current_batch_size = Tensor(\"strided_slice:0\", shape=(), dtype=int32)\n",
      "encoder_decoder_stacked_lstm_regression: number_of_features = 3\n",
      "encoder_decoder_stacked_lstm_regression: X = Tensor(\"stack:0\", shape=(2, ?, 3), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: X_sequence = [<tf.Tensor 'unstack:0' shape=(2, 3) dtype=float32>, <tf.Tensor 'unstack:1' shape=(2, 3) dtype=float32>, <tf.Tensor 'unstack:2' shape=(2, 3) dtype=float32>, <tf.Tensor 'unstack:3' shape=(2, 3) dtype=float32>, <tf.Tensor 'unstack:4' shape=(2, 3) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: lstm_cells = [<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7faf0f402390>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7faf0f696210>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7faf0f696310>]\n",
      "encoder_decoder_stacked_lstm_regression: dropout_lstm_cells = [<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x7faf0f696390>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x7faf0f6963d0>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x7faf0f696450>]\n",
      "encoder_decoder_stacked_lstm_regression: stacked_lstm_cells = <tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x7faf0f67a4d0>\n",
      "encoder_decoder_stacked_lstm_regression: encoder_cells = <tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x7faf0f696f10>\n",
      "encoder_decoder_stacked_lstm_regression: encoder_outputs = [<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_14:0' shape=(2, 16) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: encoder_final_state = (LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Add_9:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_14:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Add_9:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_14:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Add_9:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_14:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: unstacked_labels = [<tf.Tensor 'unstack_1:0' shape=(2,) dtype=float32>, <tf.Tensor 'unstack_1:1' shape=(2,) dtype=float32>, <tf.Tensor 'unstack_1:2' shape=(2,) dtype=float32>, <tf.Tensor 'unstack_1:3' shape=(2,) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: expanded_unstacked_labels = [<tf.Tensor 'ExpandDims:0' shape=(2, 1) dtype=float32>, <tf.Tensor 'ExpandDims_1:0' shape=(2, 1) dtype=float32>, <tf.Tensor 'ExpandDims_2:0' shape=(2, 1) dtype=float32>, <tf.Tensor 'ExpandDims_3:0' shape=(2, 1) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Add_9:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_14:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Add_9:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_14:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Add_9:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_14:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_1:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_2:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_1:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_2:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_1:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn/dense/Relu:0\", shape=(2, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn/dense_1/Relu:0\", shape=(2, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn/dense_2/Relu:0\", shape=(2, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: logits = Tensor(\"decoder/dnn/dense_3/BiasAdd:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: decoder_input = Tensor(\"ExpandDims_1:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_3:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_5:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_3:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_5:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_3:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_1/dense/Relu:0\", shape=(2, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_1/dense_1/Relu:0\", shape=(2, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_1/dense_2/Relu:0\", shape=(2, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: logits = Tensor(\"decoder/dnn_1/dense_3/BiasAdd:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: decoder_input = Tensor(\"ExpandDims_2:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_5:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_8:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_5:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_8:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_5:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_2/dense/Relu:0\", shape=(2, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_2/dense_1/Relu:0\", shape=(2, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_2/dense_2/Relu:0\", shape=(2, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: logits = Tensor(\"decoder/dnn_2/dense_3/BiasAdd:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: decoder_input = Tensor(\"ExpandDims_3:0\", shape=(2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0\", shape=(2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_7:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_11:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_7:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_11:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_7:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: decoder_outputs = [<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0' shape=(2, 16) dtype=float32>, <tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(2, 16) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: decoder_states = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_7:0' shape=(2, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_11:0' shape=(2, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_7:0' shape=(2, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_11:0' shape=(2, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_7:0' shape=(2, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(2, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: stacked_decoder_outputs = Tensor(\"stack_1:0\", shape=(4, 2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"stack_1:0\", shape=(4, 2, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"dnn/dense/Relu:0\", shape=(4, 2, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"dnn/dense_1/Relu:0\", shape=(4, 2, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"dnn/dense_2/Relu:0\", shape=(4, 2, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: logits = Tensor(\"dnn/dense_3/BiasAdd:0\", shape=(4, 2, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: predictions = Tensor(\"Reshape:0\", shape=(2, 4), dtype=float32)\n",
      "\n",
      "\n",
      "serving_input_fn: feature_placeholders = {'dayofweek': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>, 'hourofday': <tf.Tensor 'Placeholder_2:0' shape=(?,) dtype=string>, 'price': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>}\n",
      "serving_input_fn: fix_shape_and_type_for_serving: split_string = Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=string)\n",
      "serving_input_fn: fix_shape_and_type_for_serving: feature_tensor = Tensor(\"StringToNumber:0\", shape=(?, ?), dtype=float32)\n",
      "serving_input_fn: fix_shape_and_type_for_serving: split_string = Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=string)\n",
      "serving_input_fn: fix_shape_and_type_for_serving: feature_tensor = Tensor(\"StringToNumber_1:0\", shape=(?, ?), dtype=float32)\n",
      "serving_input_fn: fix_shape_and_type_for_serving: split_string = Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=string)\n",
      "serving_input_fn: fix_shape_and_type_for_serving: feature_tensor = Tensor(\"StringToNumber_2:0\", shape=(?, ?), dtype=float32)\n",
      "serving_input_fn: features = {'dayofweek': <tf.Tensor 'StringToNumber:0' shape=(?, ?) dtype=float32>, 'hourofday': <tf.Tensor 'StringToNumber_1:0' shape=(?, ?) dtype=float32>, 'price': <tf.Tensor 'StringToNumber_2:0' shape=(?, ?) dtype=float32>}\n",
      "serving_input_fn: get_shape_and_set_modified_shape_2D: tensor = Tensor(\"StringToNumber:0\", shape=(?, 5), dtype=float32), additional_dimension_sizes = [5]\n",
      "serving_input_fn: get_shape_and_set_modified_shape_2D: tensor = Tensor(\"StringToNumber_1:0\", shape=(?, 5), dtype=float32), additional_dimension_sizes = [5]\n",
      "serving_input_fn: get_shape_and_set_modified_shape_2D: tensor = Tensor(\"StringToNumber_2:0\", shape=(?, 5), dtype=float32), additional_dimension_sizes = [5]\n",
      "serving_input_fn: features = {'dayofweek': <tf.Tensor 'StringToNumber:0' shape=(?, 5) dtype=float32>, 'hourofday': <tf.Tensor 'StringToNumber_1:0' shape=(?, 5) dtype=float32>, 'price': <tf.Tensor 'StringToNumber_2:0' shape=(?, 5) dtype=float32>}\n",
      "serving_input_fn: sequence_lengths = Tensor(\"Tile:0\", shape=(?,), dtype=int32)\n",
      "serving_input_fn: features = {'dayofweek': <tf.Tensor 'ReverseSequence:0' shape=(?, 5) dtype=float32>, 'hourofday': <tf.Tensor 'ReverseSequence_1:0' shape=(?, 5) dtype=float32>, 'price': <tf.Tensor 'ReverseSequence_2:0' shape=(?, 5) dtype=float32>}\n",
      "\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: features = {'dayofweek': <tf.Tensor 'ReverseSequence:0' shape=(?, 5) dtype=float32>, 'hourofday': <tf.Tensor 'ReverseSequence_1:0' shape=(?, 5) dtype=float32>, 'price': <tf.Tensor 'ReverseSequence_2:0' shape=(?, 5) dtype=float32>}\n",
      "encoder_decoder_stacked_lstm_regression: labels = None\n",
      "encoder_decoder_stacked_lstm_regression: mode = infer\n",
      "encoder_decoder_stacked_lstm_regression: params = {'learning_rate': 0.01, 'lstm_dropout_output_keep_probs': [0.9, 1.0, 1.0], 'input_sequence_length': 5, 'batch_size': 2, 'dnn_hidden_units': [1024, 256, 64], 'output_sequence_length': 4, 'lstm_hidden_units': [64, 32, 16]}\n",
      "encoder_decoder_stacked_lstm_regression: current_batch_size = Tensor(\"strided_slice_4:0\", shape=(), dtype=int32)\n",
      "encoder_decoder_stacked_lstm_regression: number_of_features = 3\n",
      "encoder_decoder_stacked_lstm_regression: X = Tensor(\"stack:0\", shape=(?, 5, 3), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: X_sequence = [<tf.Tensor 'unstack:0' shape=(?, 3) dtype=float32>, <tf.Tensor 'unstack:1' shape=(?, 3) dtype=float32>, <tf.Tensor 'unstack:2' shape=(?, 3) dtype=float32>, <tf.Tensor 'unstack:3' shape=(?, 3) dtype=float32>, <tf.Tensor 'unstack:4' shape=(?, 3) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: lstm_cells = [<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7faf2359cb90>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7faf235a5150>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7faf235a5250>]\n",
      "encoder_decoder_stacked_lstm_regression: dropout_lstm_cells = [<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x7faf235a52d0>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x7faf235a5310>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x7faf235a5390>]\n",
      "encoder_decoder_stacked_lstm_regression: stacked_lstm_cells = <tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x7faf2358bd90>\n",
      "encoder_decoder_stacked_lstm_regression: encoder_cells = <tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x7faf235a5e50>\n",
      "encoder_decoder_stacked_lstm_regression: encoder_outputs = [<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0' shape=(?, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0' shape=(?, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0' shape=(?, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(?, 16) dtype=float32>, <tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_14:0' shape=(?, 16) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: encoder_final_state = (LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Add_9:0' shape=(?, 64) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_14:0' shape=(?, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Add_9:0' shape=(?, 32) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_14:0' shape=(?, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Add_9:0' shape=(?, 16) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_14:0' shape=(?, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: fake_labels = [<tf.Tensor 'zeros:0' shape=(?, 1) dtype=float32>, <tf.Tensor 'zeros_1:0' shape=(?, 1) dtype=float32>, <tf.Tensor 'zeros_2:0' shape=(?, 1) dtype=float32>, <tf.Tensor 'zeros_3:0' shape=(?, 1) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Add_9:0' shape=(?, 64) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_14:0' shape=(?, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Add_9:0' shape=(?, 32) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_14:0' shape=(?, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Add_9:0' shape=(?, 16) dtype=float32>, h=<tf.Tensor 'encoder/rnn/rnn/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_14:0' shape=(?, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0\", shape=(?, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_1:0' shape=(?, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_2:0' shape=(?, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_1:0' shape=(?, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_2:0' shape=(?, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_1:0' shape=(?, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0' shape=(?, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0\", shape=(?, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn/dense/Relu:0\", shape=(?, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn/dense_1/Relu:0\", shape=(?, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn/dense_2/Relu:0\", shape=(?, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: logits = Tensor(\"decoder/dnn/dense_3/BiasAdd:0\", shape=(?, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: decoder_input = Tensor(\"decoder/dnn/dense_3/BiasAdd:0\", shape=(?, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0\", shape=(?, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_3:0' shape=(?, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_5:0' shape=(?, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_3:0' shape=(?, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_5:0' shape=(?, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_3:0' shape=(?, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0' shape=(?, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0\", shape=(?, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_1/dense/Relu:0\", shape=(?, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_1/dense_1/Relu:0\", shape=(?, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_1/dense_2/Relu:0\", shape=(?, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: logits = Tensor(\"decoder/dnn_1/dense_3/BiasAdd:0\", shape=(?, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: decoder_input = Tensor(\"decoder/dnn_1/dense_3/BiasAdd:0\", shape=(?, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0\", shape=(?, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_5:0' shape=(?, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_8:0' shape=(?, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_5:0' shape=(?, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_8:0' shape=(?, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_5:0' shape=(?, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0' shape=(?, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0\", shape=(?, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_2/dense/Relu:0\", shape=(?, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_2/dense_1/Relu:0\", shape=(?, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: network = Tensor(\"decoder/dnn_2/dense_2/Relu:0\", shape=(?, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: logits = Tensor(\"decoder/dnn_2/dense_3/BiasAdd:0\", shape=(?, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: decoder_input = Tensor(\"decoder/dnn_2/dense_3/BiasAdd:0\", shape=(?, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: output = Tensor(\"decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0\", shape=(?, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: rnn_decoder: state = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_7:0' shape=(?, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_11:0' shape=(?, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_7:0' shape=(?, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_11:0' shape=(?, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_7:0' shape=(?, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(?, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: decoder_outputs = [<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_2:0' shape=(?, 16) dtype=float32>, <tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_5:0' shape=(?, 16) dtype=float32>, <tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_8:0' shape=(?, 16) dtype=float32>, <tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(?, 16) dtype=float32>]\n",
      "encoder_decoder_stacked_lstm_regression: decoder_states = (LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Add_7:0' shape=(?, 64) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_0/basic_lstm_cell/Mul_11:0' shape=(?, 64) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Add_7:0' shape=(?, 32) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_1/basic_lstm_cell/Mul_11:0' shape=(?, 32) dtype=float32>), LSTMStateTuple(c=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Add_7:0' shape=(?, 16) dtype=float32>, h=<tf.Tensor 'decoder/decoder/multi_rnn_cell/cell_2/basic_lstm_cell/Mul_11:0' shape=(?, 16) dtype=float32>))\n",
      "encoder_decoder_stacked_lstm_regression: stacked_decoder_outputs = Tensor(\"stack_1:0\", shape=(4, ?, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"stack_1:0\", shape=(4, ?, 16), dtype=float32)\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"dnn/dense/Relu:0\", shape=(4, ?, 1024), dtype=float32), units = 1024\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"dnn/dense_1/Relu:0\", shape=(4, ?, 256), dtype=float32), units = 256\n",
      "encoder_decoder_stacked_lstm_regression: network = Tensor(\"dnn/dense_2/Relu:0\", shape=(4, ?, 64), dtype=float32), units = 64\n",
      "encoder_decoder_stacked_lstm_regression: logits = Tensor(\"dnn/dense_3/BiasAdd:0\", shape=(4, ?, 1), dtype=float32)\n",
      "\n",
      "encoder_decoder_stacked_lstm_regression: predictions = Tensor(\"Reshape:0\", shape=(?, 4), dtype=float32)\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/envs/py2env/lib/python2.7/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_train_distribute': None, '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7faf23857e50>, '_evaluation_master': '', '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': '/content/datalab/notebooks/05-17-18/youtube-8m-team/ryan_energy_models/meta_models/trained_model/', '_global_id_in_cluster': 0, '_save_summary_steps': 100}\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after 30 secs (eval_spec.throttle_secs) or training is finished.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "2018-05-25 06:00:03.371912: I tensorflow/core/platform/cpu_feature_guard.cc:140] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into /content/datalab/notebooks/05-17-18/youtube-8m-team/ryan_energy_models/meta_models/trained_model/model.ckpt.\n",
      "INFO:tensorflow:loss = 3532.2695, step = 1\n",
      "INFO:tensorflow:global_step/sec: 43.8517\n",
      "INFO:tensorflow:loss = 16.239632, step = 101 (2.281 sec)\n",
      "INFO:tensorflow:global_step/sec: 121.613\n",
      "INFO:tensorflow:loss = 3.4313111, step = 201 (0.822 sec)\n",
      "INFO:tensorflow:global_step/sec: 117.271\n",
      "INFO:tensorflow:loss = 31.297895, step = 301 (0.853 sec)\n",
      "INFO:tensorflow:global_step/sec: 116.899\n",
      "INFO:tensorflow:loss = 98.42059, step = 401 (0.856 sec)\n",
      "INFO:tensorflow:global_step/sec: 117.029\n",
      "INFO:tensorflow:loss = 2.7823539, step = 501 (0.854 sec)\n",
      "INFO:tensorflow:global_step/sec: 118.777\n",
      "INFO:tensorflow:loss = 19.677898, step = 601 (0.842 sec)\n",
      "INFO:tensorflow:global_step/sec: 105.957\n",
      "INFO:tensorflow:loss = 3.8527641, step = 701 (0.944 sec)\n",
      "INFO:tensorflow:global_step/sec: 115.866\n",
      "INFO:tensorflow:loss = 156.24808, step = 801 (0.863 sec)\n",
      "INFO:tensorflow:global_step/sec: 115.853\n",
      "INFO:tensorflow:loss = 11.929354, step = 901 (0.863 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1000 into /content/datalab/notebooks/05-17-18/youtube-8m-team/ryan_energy_models/meta_models/trained_model/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 3.2536798.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2018-05-25-06:00:20\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /content/datalab/notebooks/05-17-18/youtube-8m-team/ryan_energy_models/meta_models/trained_model/model.ckpt-1000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2018-05-25-06:00:22\n",
      "INFO:tensorflow:Saving dict for global step 1000: global_step = 1000, loss = 22.79605, mae = 3.6498234, rmse = 4.774521\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['serving_default', 'predict_export_outputs']\n",
      "INFO:tensorflow:Restoring parameters from /content/datalab/notebooks/05-17-18/youtube-8m-team/ryan_energy_models/meta_models/trained_model/model.ckpt-1000\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: /content/datalab/notebooks/05-17-18/youtube-8m-team/ryan_energy_models/meta_models/trained_model/export/exporter/temp-1527228023/saved_model.pb\n"
     ]
    }
   ],
   "source": [
    "%bash\n",
    "rm -rf outputdir\n",
    "export PYTHONPATH=$PYTHONPATH:$PWD/model_module\n",
    "python -m trainer.task \\\n",
    "  --train_file_pattern=\"../preproc/price_data_train.csv\" \\\n",
    "  --eval_file_pattern=\"../preproc/price_data_eval.csv\"  \\\n",
    "  --output_dir=$PWD/trained_model \\\n",
    "  --batch_size=2 \\\n",
    "  --input_sequence_length=5 \\\n",
    "  --horizon=0 \\\n",
    "  --output_sequence_length=4 \\\n",
    "  --reverse_sequence=True \\\n",
    "  --model_fn_name=\"encoder_decoder_stacked_lstm_regression\" \\\n",
    "  --train_steps=1000 \\\n",
    "  --lstm_hidden_units=\"64 32 16\" \\\n",
    "  --lstm_dropout_output_keep_probs=\"0.9 1.0 1.0\" \\\n",
    "  --dnn_hidden_units=\"1024 256 64\" \\\n",
    "  --learning_rate=0.01 \\\n",
    "  --start_delay_secs=60 \\\n",
    "  --throttle_secs=30 \\\n",
    "  --job-dir=./tmp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### GCloud"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gs://ryan-asl-energy-bucket/energy_meta_models/trained_model us-central1 job_energy_meta_models180525_055932\n",
      "jobId: job_energy_meta_models180525_055932\n",
      "state: QUEUED\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "CommandException: 1 files/objects could not be removed.\n",
      "Job [job_energy_meta_models180525_055932] submitted successfully.\n",
      "Your job is still active. You may view the status of your job with the command\n",
      "\n",
      "  $ gcloud ml-engine jobs describe job_energy_meta_models180525_055932\n",
      "\n",
      "or continue streaming the logs with the command\n",
      "\n",
      "  $ gcloud ml-engine jobs stream-logs job_energy_meta_models180525_055932\n"
     ]
    }
   ],
   "source": [
    "%bash\n",
    "OUTDIR=gs://$BUCKET/energy_meta_models/trained_model\n",
    "JOBNAME=job_energy_meta_models$(date -u +%y%m%d_%H%M%S)\n",
    "echo $OUTDIR $REGION $JOBNAME\n",
    "gcloud storage rm --recursive --continue-on-error $OUTDIR\n",    "gcloud ml-engine jobs submit training $JOBNAME \\\n",
    "  --region=$REGION \\\n",
    "  --module-name=trainer.task \\\n",
    "  --package-path=$PWD/model_module/trainer \\\n",
    "  --job-dir=$OUTDIR \\\n",
    "  --staging-bucket=gs://$BUCKET \\\n",
    "  --scale-tier=STANDARD_1 \\\n",
    "  --runtime-version=1.5 \\\n",
    "  -- \\\n",
    "  --train_file_pattern=gs://$BUCKET/preproc/price_data_train.csv \\\n",
    "  --eval_file_pattern=gs://$BUCKET/preproc/price_data_eval.csv  \\\n",
    "  --output_dir=$OUTDIR \\\n",
    "  --batch_size=2 \\\n",
    "  --input_sequence_length=5 \\\n",
    "  --horizon=0 \\\n",
    "  --output_sequence_length=4 \\\n",
    "  --reverse_sequence=True \\\n",
    "  --model_fn_name=\"encoder_decoder_stacked_lstm_regression\" \\\n",
    "  --train_steps=10000 \\\n",
    "  --lstm_hidden_units=\"64 32 16\" \\\n",
    "  --lstm_dropout_output_keep_probs=\"0.9 1.0 1.0\" \\\n",
    "  --dnn_hidden_units=\"256 128 64\" \\\n",
    "  --learning_rate=0.01 \\\n",
    "  --start_delay_secs=60 \\\n",
    "  --throttle_secs=30 \\\n",
    "  --job-dir=$OUTDIR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Hyperparameter tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting hyperparam.yaml\n"
     ]
    }
   ],
   "source": [
    "%writefile hyperparam.yaml\n",
    "trainingInput:\n",
    "  scaleTier: STANDARD_1\n",
    "  hyperparameters:\n",
    "    hyperparameterMetricTag: mae\n",
    "    goal: MINIMIZE\n",
    "    maxTrials: 30\n",
    "    maxParallelTrials: 1\n",
    "    params:\n",
    "    - parameterName: batch_size\n",
    "      type: INTEGER\n",
    "      minValue: 8\n",
    "      maxValue: 512\n",
    "      scaleType: UNIT_LOG_SCALE\n",
    "    - parameterName: input_sequence_length\n",
    "      type: INTEGER\n",
    "      minValue: 10\n",
    "      maxValue: 120\n",
    "      scaleType: UNIT_LOG_SCALE\n",
    "    - parameterName: model_fn_name\n",
    "      type: CATEGORICAL\n",
    "      categoricalValues: [\"dnn_regression\",\"stacked_lstm_regression\",\"stacked_lstmN_regression\",\"encoder_decoder_stacked_lstm_regression\"]\n",
    "    - parameterName: lstm_hidden_units\n",
    "      type: CATEGORICAL\n",
    "      categoricalValues: [\"64 32 16\", \"256 128 16\", \"64 64 64\"]\n",
    "    - parameterName: lstm_dropout_output_keep_probs\n",
    "      type: CATEGORICAL\n",
    "      categoricalValues: [\"0.9 1.0 1.0\", \"0.95 0.95 1.0\", \"0.95 0.95 0.95\"]\n",
    "    - parameterName: dnn_hidden_units\n",
    "      type: CATEGORICAL\n",
    "      categoricalValues: [\"256 128 64\", \"256 128 16\", \"64 64 64\"]\n",
    "    - parameterName: learning_rate\n",
    "      type: DOUBLE\n",
    "      minValue: 0.001\n",
    "      maxValue: 0.1\n",
    "      scaleType: UNIT_LINEAR_SCALE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gs://ryan-asl-energy-bucket/energy_meta_models/hyperparam us-central1 job_energy_meta_models180525_113731\n",
      "jobId: job_energy_meta_models180525_113731\n",
      "state: QUEUED\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "CommandException: 1 files/objects could not be removed.\n",
      "Job [job_energy_meta_models180525_113731] submitted successfully.\n",
      "Your job is still active. You may view the status of your job with the command\n",
      "\n",
      "  $ gcloud ml-engine jobs describe job_energy_meta_models180525_113731\n",
      "\n",
      "or continue streaming the logs with the command\n",
      "\n",
      "  $ gcloud ml-engine jobs stream-logs job_energy_meta_models180525_113731\n"
     ]
    }
   ],
   "source": [
    "%bash\n",
    "OUTDIR=gs://$BUCKET/energy_meta_models/hyperparam\n",
    "JOBNAME=job_energy_meta_models$(date -u +%y%m%d_%H%M%S)\n",
    "echo $OUTDIR $REGION $JOBNAME\n",
    "gcloud storage rm --recursive --continue-on-error $OUTDIR\n",    "gcloud ml-engine jobs submit training $JOBNAME \\\n",
    "  --region=$REGION \\\n",
    "  --module-name=trainer.task \\\n",
    "  --package-path=$PWD/model_module/trainer \\\n",
    "  --job-dir=$OUTDIR \\\n",
    "  --staging-bucket=gs://$BUCKET \\\n",
    "  --scale-tier=STANDARD_1 \\\n",
    "  --config=hyperparam.yaml \\\n",
    "  --runtime-version=1.5 \\\n",
    "  -- \\\n",
    "  --train_file_pattern=gs://$BUCKET/preproc/price_data_train.csv \\\n",
    "  --eval_file_pattern=gs://$BUCKET/preproc/price_data_eval.csv  \\\n",
    "  --output_dir=$OUTDIR \\\n",
    "  --horizon=0 \\\n",
    "  --output_sequence_length=4 \\\n",
    "  --reverse_sequence=True \\\n",
    "  --train_steps=10000 \\\n",
    "  --start_delay_secs=60 \\\n",
    "  --throttle_secs=30 \\\n",
    "  --job-dir=$OUTDIR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Deploy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deleting and deploying energy_meta_models v1 from gs://ryan-asl-energy-bucket/energy_meta_models/trained_model/export/exporter/1527228207/ ... this will take a few minutes\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Created ml engine model [projects/qwiklabs-gcp-8d3d0cd07cef9252/models/energy_meta_models].\n",
      "Creating version (this might take a few minutes)......\n",
      ".......................................................................................................done.\n"
     ]
    }
   ],
   "source": [
    "%bash\n",
    "MODEL_NAME=\"energy_meta_models\"\n",
    "MODEL_VERSION=\"v1\"\n",
    "MODEL_LOCATION=$(gcloud storage ls gs://$BUCKET/energy_meta_models/trained_model/export/exporter/ | tail -1)\n",
    "echo \"Deleting and deploying $MODEL_NAME $MODEL_VERSION from $MODEL_LOCATION ... this will take a few minutes\"\n",
    "#gcloud ml-engine versions delete ${MODEL_VERSION} --model ${MODEL_NAME}\n",
    "#gcloud ml-engine models delete ${MODEL_NAME}\n",
    "gcloud ml-engine models create $MODEL_NAME --regions $REGION\n",
    "gcloud ml-engine versions create $MODEL_VERSION --model $MODEL_NAME --origin $MODEL_LOCATION --runtime-version 1.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Prediction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "evaldf = pd.read_csv(filepath_or_buffer = \"../preproc/price_data_eval.csv\", sep = \",\", names = [\"price\", \"dayofweek\", \"hourofday\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>price</th>\n",
       "      <th>dayofweek</th>\n",
       "      <th>hourofday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>45.89</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>45.50</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>46.85</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>48.54</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>48.54</td>\n",
       "      <td>6</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   price  dayofweek  hourofday\n",
       "0  45.89          6          9\n",
       "1  45.50          6         10\n",
       "2  46.85          6         11\n",
       "3  48.54          6         12\n",
       "4  48.54          6         13"
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaldf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max_sequences = 1903\n"
     ]
    }
   ],
   "source": [
    "max_sequences = len(evaldf) - arguments['input_sequence_length'] - arguments['horizon'] - arguments['output_sequence_length'] + 1\n",
    "print(\"max_sequences = {}\".format(max_sequences))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "sequences_dict = {\"price\": [], \"dayofweek\": [], \"hourofday\": [], \"labels\": []}\n",
    "for i in xrange(max_sequences):\n",
    "  sequences_dict[\"price\"].append(str(evaldf[\"price\"][i:i + arguments['input_sequence_length']].tolist()))\n",
    "  sequences_dict[\"dayofweek\"].append(str(evaldf[\"dayofweek\"][i:i + arguments['input_sequence_length']].tolist()))\n",
    "  sequences_dict[\"hourofday\"].append(str(evaldf[\"hourofday\"][i:i + arguments['input_sequence_length']].tolist()))\n",
    "  sequences_dict[\"labels\"].append(str(evaldf[\"price\"][i + arguments['input_sequence_length'] + arguments['horizon']:i + arguments['input_sequence_length'] + arguments['horizon'] + arguments['output_sequence_length']].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>price</th>\n",
       "      <th>dayofweek</th>\n",
       "      <th>hourofday</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[45.89, 45.5, 46.85, 48.54, 48.54]</td>\n",
       "      <td>[6, 6, 6, 6, 6]</td>\n",
       "      <td>[9, 10, 11, 12, 13]</td>\n",
       "      <td>[44.92, 44.76, 48.54, 57.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[45.5, 46.85, 48.54, 48.54, 44.92]</td>\n",
       "      <td>[6, 6, 6, 6, 6]</td>\n",
       "      <td>[10, 11, 12, 13, 14]</td>\n",
       "      <td>[44.76, 48.54, 57.0, 60.01]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[46.85, 48.54, 48.54, 44.92, 44.76]</td>\n",
       "      <td>[6, 6, 6, 6, 6]</td>\n",
       "      <td>[11, 12, 13, 14, 15]</td>\n",
       "      <td>[48.54, 57.0, 60.01, 60.48]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[48.54, 48.54, 44.92, 44.76, 48.54]</td>\n",
       "      <td>[6, 6, 6, 6, 6]</td>\n",
       "      <td>[12, 13, 14, 15, 16]</td>\n",
       "      <td>[57.0, 60.01, 60.48, 60.3]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[48.54, 44.92, 44.76, 48.54, 57.0]</td>\n",
       "      <td>[6, 6, 6, 6, 6]</td>\n",
       "      <td>[13, 14, 15, 16, 17]</td>\n",
       "      <td>[60.01, 60.48, 60.3, 58.02]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 price        dayofweek             hourofday  \\\n",
       "0   [45.89, 45.5, 46.85, 48.54, 48.54]  [6, 6, 6, 6, 6]   [9, 10, 11, 12, 13]   \n",
       "1   [45.5, 46.85, 48.54, 48.54, 44.92]  [6, 6, 6, 6, 6]  [10, 11, 12, 13, 14]   \n",
       "2  [46.85, 48.54, 48.54, 44.92, 44.76]  [6, 6, 6, 6, 6]  [11, 12, 13, 14, 15]   \n",
       "3  [48.54, 48.54, 44.92, 44.76, 48.54]  [6, 6, 6, 6, 6]  [12, 13, 14, 15, 16]   \n",
       "4   [48.54, 44.92, 44.76, 48.54, 57.0]  [6, 6, 6, 6, 6]  [13, 14, 15, 16, 17]   \n",
       "\n",
       "                        labels  \n",
       "0  [44.92, 44.76, 48.54, 57.0]  \n",
       "1  [44.76, 48.54, 57.0, 60.01]  \n",
       "2  [48.54, 57.0, 60.01, 60.48]  \n",
       "3   [57.0, 60.01, 60.48, 60.3]  \n",
       "4  [60.01, 60.48, 60.3, 58.02]  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sequences_dict = pd.DataFrame.from_dict(sequences_dict)\n",
    "sequences_dict = sequences_dict[[\"price\",\"dayofweek\",\"hourofday\",\"labels\"]]\n",
    "sequences_dict.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "Load is 100% Complete\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import datalab.bigquery as bq\n",
    "bigquery_dataset_name = 'ryan_energy'\n",
    "bigquery_table_name = 'tbl_eval_sequences'\n",
    "\n",
    "# Define BigQuery dataset and table\\n\",\n",
    "dataset = bq.Dataset(bigquery_dataset_name)\n",
    "table = bq.Table(bigquery_dataset_name + '.' + bigquery_table_name)\n",
    "\n",
    "# Create BigQuery dataset\n",
    "if not dataset.exists():\n",
    "  dataset.create()\n",
    "\n",
    "# Create or overwrite the existing table if it exists\\n\",\n",
    "table_schema = bq.Schema.from_dataframe(sequences_dict)\n",
    "table.create(schema = table_schema, overwrite = True)\n",
    "\n",
    "sequences_dict.to_gbq(destination_table = bigquery_dataset_name + '.' + bigquery_table_name, project_id = \"qwiklabs-gcp-8d3d0cd07cef9252\", if_exists = \"replace\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Create SQL query\n",
    "query=\"\"\"\n",
    "SELECT\n",
    "  price,\n",
    "  dayofweek,\n",
    "  hourofday\n",
    "FROM\n",
    "  `qwiklabs-gcp-8d3d0cd07cef9252.ryan_energy.tbl_eval_sequences`\n",
    "LIMIT\n",
    "  3\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>price</th>\n",
       "      <th>dayofweek</th>\n",
       "      <th>hourofday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[49.1, 45.1, 43.15, 43.15, 43.39]</td>\n",
       "      <td>[0, 0, 0, 0, 0]</td>\n",
       "      <td>[0, 1, 2, 3, 4]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[42.28, 39.21, 36.0, 35.13, 40.0]</td>\n",
       "      <td>[0, 0, 0, 0, 0]</td>\n",
       "      <td>[0, 1, 2, 3, 4]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[53.45, 49.34, 48.42, 48.15, 48.42]</td>\n",
       "      <td>[0, 0, 0, 0, 0]</td>\n",
       "      <td>[0, 1, 2, 3, 4]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 price        dayofweek        hourofday\n",
       "0    [49.1, 45.1, 43.15, 43.15, 43.39]  [0, 0, 0, 0, 0]  [0, 1, 2, 3, 4]\n",
       "1    [42.28, 39.21, 36.0, 35.13, 40.0]  [0, 0, 0, 0, 0]  [0, 1, 2, 3, 4]\n",
       "2  [53.45, 49.34, 48.42, 48.15, 48.42]  [0, 0, 0, 0, 0]  [0, 1, 2, 3, 4]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import google.datalab.bigquery as bq2\n",
    "df_predict = bq2.Query(query).execute().result().to_dataframe()\n",
    "df_predict.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Local prediction from local model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with open('sequences.json', 'w') as outfile:\n",
    "  for idx, row in df_predict.iterrows():\n",
    "    json_string = \"{\\\"price\\\": \\\"\" + row[\"price\"].replace(\" \",\"\").replace(\"[\",\"\").replace(\"]\",\"\") + \"\\\", \\\"dayofweek\\\": \\\"\" + row[\"dayofweek\"].replace(\" \",\"\").replace(\"[\",\"\").replace(\"]\",\"\") + \"\\\", \\\"hourofday\\\": \\\"\" + row[\"hourofday\"].replace(\" \",\"\").replace(\"[\",\"\").replace(\"]\",\"\") + \"\\\"}\"\n",
    "    outfile.write(\"%s\\n\" % json_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OUTPUT\n",
      "[13.399503707885742, 13.564470291137695, 13.342259407043457, 12.674761772155762]\n",
      "[12.554098129272461, 12.64464282989502, 11.737497329711914, 11.962775230407715]\n",
      "[11.88149642944336, 11.543296813964844, 11.965831756591797, 11.754377365112305]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: /usr/local/envs/py2env/lib/python2.7/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "2018-05-25 05:53:42.594606: I tensorflow/core/platform/cpu_feature_guard.cc:140] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%bash\n",
    "model_dir=$(ls ${PWD}/trained_model/export/exporter | tail -1)\n",
    "gcloud ml-engine local predict \\\n",
    "    --model-dir=${PWD}/trained_model/export/exporter/${model_dir} \\\n",
    "    --json-instances=./sequences.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### GCloud ML-Engine prediction from deployed model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Format dataframe to instances list to get sent to ML-Engine\n",
    "instances = [{\"price\": row[\"price\"].replace(\" \",\"\").replace(\"[\",\"\").replace(\"]\",\"\"), \"dayofweek\": row[\"dayofweek\"].replace(\" \",\"\").replace(\"[\",\"\").replace(\"]\",\"\"), \"hourofday\": row[\"hourofday\"].replace(\" \",\"\").replace(\"[\",\"\").replace(\"]\",\"\")} for idx, row in df_predict.iterrows()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "response = {u'predictions': [{u'output': [25.096506118774414, 25.066381454467773, 25.037992477416992, 23.214223861694336]}, {u'output': [23.233545303344727, 23.211305618286133, 23.10343360900879, 23.093595504760742]}, {u'output': [23.08864974975586, 23.078981399536133, 23.08859634399414, 23.399633407592773]}]}\n"
     ]
    }
   ],
   "source": [
    "# Send instance dictionary to receive response from ML-Engine for online prediction\n",
    "from googleapiclient import discovery\n",
    "from oauth2client.client import GoogleCredentials\n",
    "import json\n",
    "\n",
    "credentials = GoogleCredentials.get_application_default()\n",
    "api = discovery.build('ml', 'v1', credentials=credentials)\n",
    "\n",
    "request_data = {\"instances\": instances}\n",
    "\n",
    "parent = 'projects/%s/models/%s/versions/%s' % (PROJECT, 'energy_meta_models', 'v1')\n",
    "response = api.projects().predict(body = request_data, name = parent).execute()\n",
    "print(\"response = {}\".format(response))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
